2024-11-11 22:16:21,620 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 22:16:21,630 main DEBUG Took 0.008955 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 22:16:21,631 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 22:16:21,631 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 22:16:21,632 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 22:16:21,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,639 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 22:16:21,650 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,651 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,652 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,652 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,652 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,654 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,654 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,654 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,655 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,655 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,656 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,657 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,657 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,657 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,658 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,658 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,659 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,659 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,660 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,660 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,661 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 22:16:21,661 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,662 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 22:16:21,663 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 22:16:21,664 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 22:16:21,667 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 22:16:21,668 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 22:16:21,669 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 22:16:21,670 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 22:16:21,681 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 22:16:21,684 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 22:16:21,686 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 22:16:21,686 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 22:16:21,687 main DEBUG createAppenders(={Console}) 2024-11-11 22:16:21,688 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 22:16:21,688 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 22:16:21,689 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 22:16:21,689 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 22:16:21,690 main DEBUG OutputStream closed 2024-11-11 22:16:21,690 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 22:16:21,691 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 22:16:21,691 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 22:16:21,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 22:16:21,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 22:16:21,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 22:16:21,782 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 22:16:21,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 22:16:21,783 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 22:16:21,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 22:16:21,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 22:16:21,784 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 22:16:21,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 22:16:21,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 22:16:21,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 22:16:21,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 22:16:21,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 22:16:21,786 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 22:16:21,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 22:16:21,787 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 22:16:21,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 22:16:21,790 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 22:16:21,790 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 22:16:21,791 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 22:16:21,792 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T22:16:22,018 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949 2024-11-11 22:16:22,021 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 22:16:22,021 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T22:16:22,030 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-11T22:16:22,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T22:16:22,086 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8, deleteOnExit=true 2024-11-11T22:16:22,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T22:16:22,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/test.cache.data in system properties and HBase conf 2024-11-11T22:16:22,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T22:16:22,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/hadoop.log.dir in system properties and HBase conf 2024-11-11T22:16:22,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T22:16:22,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T22:16:22,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T22:16:22,181 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T22:16:22,269 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T22:16:22,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T22:16:22,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T22:16:22,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T22:16:22,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T22:16:22,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T22:16:22,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T22:16:22,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T22:16:22,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T22:16:22,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T22:16:22,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/nfs.dump.dir in system properties and HBase conf 2024-11-11T22:16:22,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/java.io.tmpdir in system properties and HBase conf 2024-11-11T22:16:22,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T22:16:22,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T22:16:22,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T22:16:23,408 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T22:16:23,479 INFO [Time-limited test {}] log.Log(170): Logging initialized @2718ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T22:16:23,546 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T22:16:23,599 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T22:16:23,621 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T22:16:23,621 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T22:16:23,622 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T22:16:23,632 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T22:16:23,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/hadoop.log.dir/,AVAILABLE} 2024-11-11T22:16:23,636 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T22:16:23,853 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/java.io.tmpdir/jetty-localhost-41511-hadoop-hdfs-3_4_1-tests_jar-_-any-9748471090780142807/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T22:16:23,863 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:41511} 2024-11-11T22:16:23,864 INFO [Time-limited test {}] server.Server(415): Started @3104ms 2024-11-11T22:16:24,574 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T22:16:24,585 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T22:16:24,587 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T22:16:24,587 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T22:16:24,588 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T22:16:24,588 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/hadoop.log.dir/,AVAILABLE} 2024-11-11T22:16:24,589 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T22:16:24,694 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/java.io.tmpdir/jetty-localhost-44407-hadoop-hdfs-3_4_1-tests_jar-_-any-2936695406231477077/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T22:16:24,695 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:44407} 2024-11-11T22:16:24,695 INFO [Time-limited test {}] server.Server(415): Started @3935ms 2024-11-11T22:16:24,747 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T22:16:26,101 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/data/data2/current/BP-2142985488-172.17.0.3-1731363382824/current, will proceed with Du for space computation calculation, 2024-11-11T22:16:26,101 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/data/data1/current/BP-2142985488-172.17.0.3-1731363382824/current, will proceed with Du for space computation calculation, 2024-11-11T22:16:26,130 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T22:16:26,187 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4b84a186c4ed0d20 with lease ID 0x12f0e1f40675ea24: Processing first storage report for DS-73559dc3-e70a-413a-b60c-4043cca0a565 from datanode DatanodeRegistration(127.0.0.1:44663, datanodeUuid=1380727f-fd36-4c3d-bafb-8685d0ba11cc, infoPort=34927, infoSecurePort=0, ipcPort=40341, storageInfo=lv=-57;cid=testClusterID;nsid=656434497;c=1731363382825) 2024-11-11T22:16:26,189 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4b84a186c4ed0d20 with lease ID 0x12f0e1f40675ea24: from storage DS-73559dc3-e70a-413a-b60c-4043cca0a565 node DatanodeRegistration(127.0.0.1:44663, datanodeUuid=1380727f-fd36-4c3d-bafb-8685d0ba11cc, infoPort=34927, infoSecurePort=0, ipcPort=40341, storageInfo=lv=-57;cid=testClusterID;nsid=656434497;c=1731363382825), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T22:16:26,189 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4b84a186c4ed0d20 with lease ID 0x12f0e1f40675ea24: Processing first storage report for DS-5d34d3e1-1eec-4677-acb3-66bb3e0aec99 from datanode DatanodeRegistration(127.0.0.1:44663, datanodeUuid=1380727f-fd36-4c3d-bafb-8685d0ba11cc, infoPort=34927, infoSecurePort=0, ipcPort=40341, storageInfo=lv=-57;cid=testClusterID;nsid=656434497;c=1731363382825) 2024-11-11T22:16:26,189 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4b84a186c4ed0d20 with lease ID 0x12f0e1f40675ea24: from storage DS-5d34d3e1-1eec-4677-acb3-66bb3e0aec99 node DatanodeRegistration(127.0.0.1:44663, datanodeUuid=1380727f-fd36-4c3d-bafb-8685d0ba11cc, infoPort=34927, infoSecurePort=0, ipcPort=40341, storageInfo=lv=-57;cid=testClusterID;nsid=656434497;c=1731363382825), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T22:16:26,253 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949 2024-11-11T22:16:26,325 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/zookeeper_0, clientPort=54105, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T22:16:26,335 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54105 2024-11-11T22:16:26,349 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:16:26,352 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:16:26,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741825_1001 (size=7) 2024-11-11T22:16:26,981 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e with version=8 2024-11-11T22:16:26,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/hbase-staging 2024-11-11T22:16:27,088 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T22:16:27,337 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8d82b89dd27d:0 server-side Connection retries=45 2024-11-11T22:16:27,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T22:16:27,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T22:16:27,351 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T22:16:27,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T22:16:27,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T22:16:27,500 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T22:16:27,552 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T22:16:27,561 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T22:16:27,564 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T22:16:27,589 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8299 (auto-detected) 2024-11-11T22:16:27,590 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-11T22:16:27,609 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41919 2024-11-11T22:16:27,631 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41919 connecting to ZooKeeper ensemble=127.0.0.1:54105 2024-11-11T22:16:27,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:419190x0, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T22:16:27,752 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41919-0x1012bf8af6e0000 connected 2024-11-11T22:16:27,852 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:16:27,855 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:16:27,866 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T22:16:27,870 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e, hbase.cluster.distributed=false 2024-11-11T22:16:27,899 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T22:16:27,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41919 2024-11-11T22:16:27,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41919 2024-11-11T22:16:27,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41919 2024-11-11T22:16:27,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41919 2024-11-11T22:16:27,907 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41919 2024-11-11T22:16:28,030 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8d82b89dd27d:0 server-side Connection retries=45 2024-11-11T22:16:28,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T22:16:28,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T22:16:28,032 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T22:16:28,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T22:16:28,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T22:16:28,035 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T22:16:28,039 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T22:16:28,040 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36911 2024-11-11T22:16:28,044 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36911 connecting to ZooKeeper ensemble=127.0.0.1:54105 2024-11-11T22:16:28,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:16:28,050 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:16:28,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369110x0, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T22:16:28,067 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:369110x0, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T22:16:28,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36911-0x1012bf8af6e0001 connected 2024-11-11T22:16:28,070 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T22:16:28,078 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T22:16:28,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T22:16:28,088 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T22:16:28,089 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36911 2024-11-11T22:16:28,090 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36911 2024-11-11T22:16:28,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36911 2024-11-11T22:16:28,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36911 2024-11-11T22:16:28,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36911 2024-11-11T22:16:28,116 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8d82b89dd27d:41919 2024-11-11T22:16:28,117 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:28,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T22:16:28,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T22:16:28,136 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:28,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T22:16:28,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:28,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:28,167 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T22:16:28,168 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8d82b89dd27d,41919,1731363387169 from backup master directory 2024-11-11T22:16:28,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T22:16:28,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:28,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T22:16:28,183 WARN [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T22:16:28,183 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:28,185 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T22:16:28,186 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T22:16:28,239 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/hbase.id] with ID: 804d1099-6056-4e2e-b24a-46ac147d80e8 2024-11-11T22:16:28,240 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/.tmp/hbase.id 2024-11-11T22:16:28,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741826_1002 (size=42) 2024-11-11T22:16:28,655 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/.tmp/hbase.id]:[hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/hbase.id] 2024-11-11T22:16:28,704 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T22:16:28,710 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T22:16:28,736 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-11T22:16:28,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:28,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:28,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741827_1003 (size=196) 2024-11-11T22:16:29,229 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:16:29,232 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T22:16:29,246 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:29,250 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T22:16:29,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741828_1004 (size=1189) 2024-11-11T22:16:29,703 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store 2024-11-11T22:16:29,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741829_1005 (size=34) 2024-11-11T22:16:30,135 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T22:16:30,139 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:16:30,140 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T22:16:30,140 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:16:30,140 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:16:30,142 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T22:16:30,142 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:16:30,142 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:16:30,143 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731363390140Disabling compacts and flushes for region at 1731363390140Disabling writes for close at 1731363390142 (+2 ms)Writing region close event to WAL at 1731363390142Closed at 1731363390142 2024-11-11T22:16:30,146 WARN [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/.initializing 2024-11-11T22:16:30,146 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/WALs/8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:30,153 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T22:16:30,167 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8d82b89dd27d%2C41919%2C1731363387169, suffix=, logDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/WALs/8d82b89dd27d,41919,1731363387169, archiveDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/oldWALs, maxLogs=10 2024-11-11T22:16:30,187 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/WALs/8d82b89dd27d,41919,1731363387169/8d82b89dd27d%2C41919%2C1731363387169.1731363390170, exclude list is [], retry=0 2024-11-11T22:16:30,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44663,DS-73559dc3-e70a-413a-b60c-4043cca0a565,DISK] 2024-11-11T22:16:30,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T22:16:30,235 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/WALs/8d82b89dd27d,41919,1731363387169/8d82b89dd27d%2C41919%2C1731363387169.1731363390170 2024-11-11T22:16:30,236 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34927:34927)] 2024-11-11T22:16:30,237 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:16:30,238 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:16:30,241 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,242 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,299 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T22:16:30,302 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:30,304 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:30,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,308 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T22:16:30,309 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:30,310 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:16:30,310 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,313 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T22:16:30,313 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:30,314 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:16:30,314 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,317 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T22:16:30,317 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:30,318 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:16:30,318 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,322 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,323 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,328 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,328 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,332 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T22:16:30,336 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T22:16:30,342 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:16:30,344 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61065356, jitterRate=-0.09005528688430786}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T22:16:30,351 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731363390254Initializing all the Stores at 1731363390256 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363390256Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363390257 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363390257Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363390257Cleaning up temporary data from old regions at 1731363390328 (+71 ms)Region opened successfully at 1731363390351 (+23 ms) 2024-11-11T22:16:30,352 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T22:16:30,380 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32964c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8d82b89dd27d/172.17.0.3:0 2024-11-11T22:16:30,406 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T22:16:30,415 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T22:16:30,415 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T22:16:30,418 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T22:16:30,419 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T22:16:30,423 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T22:16:30,423 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T22:16:30,450 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T22:16:30,458 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T22:16:30,470 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T22:16:30,473 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T22:16:30,474 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T22:16:30,487 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T22:16:30,490 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T22:16:30,494 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T22:16:30,502 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T22:16:30,504 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T22:16:30,512 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T22:16:30,536 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T22:16:30,544 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T22:16:30,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T22:16:30,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T22:16:30,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:30,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:30,557 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8d82b89dd27d,41919,1731363387169, sessionid=0x1012bf8af6e0000, setting cluster-up flag (Was=false) 2024-11-11T22:16:30,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:30,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:30,618 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T22:16:30,622 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:30,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:30,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:30,676 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T22:16:30,679 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:30,691 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T22:16:30,707 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(746): ClusterId : 804d1099-6056-4e2e-b24a-46ac147d80e8 2024-11-11T22:16:30,709 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T22:16:30,725 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T22:16:30,725 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T22:16:30,735 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T22:16:30,735 DEBUG [RS:0;8d82b89dd27d:36911 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631848d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8d82b89dd27d/172.17.0.3:0 2024-11-11T22:16:30,750 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8d82b89dd27d:36911 2024-11-11T22:16:30,753 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T22:16:30,753 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T22:16:30,753 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T22:16:30,756 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(2659): reportForDuty to master=8d82b89dd27d,41919,1731363387169 with port=36911, startcode=1731363387997 2024-11-11T22:16:30,766 DEBUG [RS:0;8d82b89dd27d:36911 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T22:16:30,766 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T22:16:30,775 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T22:16:30,782 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T22:16:30,787 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8d82b89dd27d,41919,1731363387169 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T22:16:30,793 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8d82b89dd27d:0, corePoolSize=5, maxPoolSize=5 2024-11-11T22:16:30,794 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8d82b89dd27d:0, corePoolSize=5, maxPoolSize=5 2024-11-11T22:16:30,794 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=5, maxPoolSize=5 2024-11-11T22:16:30,794 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=5, maxPoolSize=5 2024-11-11T22:16:30,794 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8d82b89dd27d:0, corePoolSize=10, maxPoolSize=10 2024-11-11T22:16:30,794 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:30,795 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=2, maxPoolSize=2 2024-11-11T22:16:30,795 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:30,798 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731363420798 2024-11-11T22:16:30,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38669, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T22:16:30,799 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T22:16:30,800 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T22:16:30,801 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T22:16:30,801 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T22:16:30,805 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T22:16:30,805 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T22:16:30,805 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41919 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:30,805 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T22:16:30,806 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T22:16:30,807 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:30,807 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T22:16:30,806 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:30,810 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T22:16:30,812 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T22:16:30,812 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T22:16:30,816 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T22:16:30,817 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T22:16:30,819 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.large.0-1731363390818,5,FailOnTimeoutGroup] 2024-11-11T22:16:30,820 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.small.0-1731363390820,5,FailOnTimeoutGroup] 2024-11-11T22:16:30,820 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:30,821 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T22:16:30,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741831_1007 (size=1321) 2024-11-11T22:16:30,822 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:30,823 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:30,832 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-11T22:16:30,832 WARN [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T22:16:30,935 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(2659): reportForDuty to master=8d82b89dd27d,41919,1731363387169 with port=36911, startcode=1731363387997 2024-11-11T22:16:30,942 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41919 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:30,945 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41919 {}] master.ServerManager(517): Registering regionserver=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:30,955 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e 2024-11-11T22:16:30,955 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37063 2024-11-11T22:16:30,955 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T22:16:30,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T22:16:30,966 DEBUG [RS:0;8d82b89dd27d:36911 {}] zookeeper.ZKUtil(111): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:30,966 WARN [RS:0;8d82b89dd27d:36911 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T22:16:30,966 INFO [RS:0;8d82b89dd27d:36911 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T22:16:30,967 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/WALs/8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:30,969 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8d82b89dd27d,36911,1731363387997] 2024-11-11T22:16:30,993 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T22:16:31,007 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T22:16:31,011 INFO [RS:0;8d82b89dd27d:36911 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T22:16:31,012 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,013 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T22:16:31,017 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T22:16:31,018 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,018 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,019 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,019 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,019 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,019 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,019 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8d82b89dd27d:0, corePoolSize=2, maxPoolSize=2 2024-11-11T22:16:31,019 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,019 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,020 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,020 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,020 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,020 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8d82b89dd27d:0, corePoolSize=1, maxPoolSize=1 2024-11-11T22:16:31,020 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8d82b89dd27d:0, corePoolSize=3, maxPoolSize=3 2024-11-11T22:16:31,020 DEBUG [RS:0;8d82b89dd27d:36911 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0, corePoolSize=3, maxPoolSize=3 2024-11-11T22:16:31,021 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,021 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,022 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,022 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,022 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,022 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,36911,1731363387997-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T22:16:31,041 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T22:16:31,043 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,36911,1731363387997-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,043 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,043 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.Replication(171): 8d82b89dd27d,36911,1731363387997 started 2024-11-11T22:16:31,061 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:31,061 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1482): Serving as 8d82b89dd27d,36911,1731363387997, RpcServer on 8d82b89dd27d/172.17.0.3:36911, sessionid=0x1012bf8af6e0001 2024-11-11T22:16:31,062 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T22:16:31,062 DEBUG [RS:0;8d82b89dd27d:36911 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:31,062 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8d82b89dd27d,36911,1731363387997' 2024-11-11T22:16:31,062 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T22:16:31,063 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T22:16:31,064 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T22:16:31,064 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T22:16:31,064 DEBUG [RS:0;8d82b89dd27d:36911 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:31,064 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8d82b89dd27d,36911,1731363387997' 2024-11-11T22:16:31,064 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T22:16:31,065 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T22:16:31,066 DEBUG [RS:0;8d82b89dd27d:36911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T22:16:31,066 INFO [RS:0;8d82b89dd27d:36911 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T22:16:31,066 INFO [RS:0;8d82b89dd27d:36911 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T22:16:31,176 INFO [RS:0;8d82b89dd27d:36911 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T22:16:31,179 INFO [RS:0;8d82b89dd27d:36911 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8d82b89dd27d%2C36911%2C1731363387997, suffix=, logDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/WALs/8d82b89dd27d,36911,1731363387997, archiveDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/oldWALs, maxLogs=32 2024-11-11T22:16:31,194 DEBUG [RS:0;8d82b89dd27d:36911 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/WALs/8d82b89dd27d,36911,1731363387997/8d82b89dd27d%2C36911%2C1731363387997.1731363391181, exclude list is [], retry=0 2024-11-11T22:16:31,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44663,DS-73559dc3-e70a-413a-b60c-4043cca0a565,DISK] 2024-11-11T22:16:31,202 INFO [RS:0;8d82b89dd27d:36911 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/WALs/8d82b89dd27d,36911,1731363387997/8d82b89dd27d%2C36911%2C1731363387997.1731363391181 2024-11-11T22:16:31,202 DEBUG [RS:0;8d82b89dd27d:36911 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34927:34927)] 2024-11-11T22:16:31,224 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T22:16:31,224 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e 2024-11-11T22:16:31,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741833_1009 (size=32) 2024-11-11T22:16:31,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:16:31,644 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T22:16:31,647 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T22:16:31,648 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:31,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:31,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T22:16:31,652 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T22:16:31,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:31,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:31,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T22:16:31,656 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T22:16:31,656 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:31,657 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:31,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T22:16:31,661 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T22:16:31,661 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:31,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:31,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T22:16:31,664 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740 2024-11-11T22:16:31,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740 2024-11-11T22:16:31,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T22:16:31,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T22:16:31,669 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:16:31,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T22:16:31,676 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:16:31,676 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70427712, jitterRate=0.049454689025878906}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:16:31,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731363391640Initializing all the Stores at 1731363391643 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363391643Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363391643Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363391644 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363391644Cleaning up temporary data from old regions at 1731363391668 (+24 ms)Region opened successfully at 1731363391679 (+11 ms) 2024-11-11T22:16:31,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T22:16:31,679 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T22:16:31,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T22:16:31,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T22:16:31,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T22:16:31,681 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T22:16:31,681 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731363391679Disabling compacts and flushes for region at 1731363391679Disabling writes for close at 1731363391679Writing region close event to WAL at 1731363391680 (+1 ms)Closed at 1731363391681 (+1 ms) 2024-11-11T22:16:31,690 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T22:16:31,690 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T22:16:31,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T22:16:31,711 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T22:16:31,714 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T22:16:31,868 DEBUG [8d82b89dd27d:41919 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T22:16:31,880 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:31,887 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8d82b89dd27d,36911,1731363387997, state=OPENING 2024-11-11T22:16:31,902 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T22:16:31,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:31,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:16:31,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T22:16:31,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T22:16:31,915 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T22:16:31,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:16:32,097 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T22:16:32,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45161, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T22:16:32,112 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T22:16:32,112 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T22:16:32,113 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T22:16:32,117 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8d82b89dd27d%2C36911%2C1731363387997.meta, suffix=.meta, logDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/WALs/8d82b89dd27d,36911,1731363387997, archiveDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/oldWALs, maxLogs=32 2024-11-11T22:16:32,134 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/WALs/8d82b89dd27d,36911,1731363387997/8d82b89dd27d%2C36911%2C1731363387997.meta.1731363392120.meta, exclude list is [], retry=0 2024-11-11T22:16:32,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44663,DS-73559dc3-e70a-413a-b60c-4043cca0a565,DISK] 2024-11-11T22:16:32,141 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/WALs/8d82b89dd27d,36911,1731363387997/8d82b89dd27d%2C36911%2C1731363387997.meta.1731363392120.meta 2024-11-11T22:16:32,141 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34927:34927)] 2024-11-11T22:16:32,142 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:16:32,143 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T22:16:32,146 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T22:16:32,150 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T22:16:32,154 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T22:16:32,155 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:16:32,155 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T22:16:32,155 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T22:16:32,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T22:16:32,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T22:16:32,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:32,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:32,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T22:16:32,164 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T22:16:32,164 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:32,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:32,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T22:16:32,168 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T22:16:32,168 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:32,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:32,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T22:16:32,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T22:16:32,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:32,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T22:16:32,172 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T22:16:32,173 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740 2024-11-11T22:16:32,176 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740 2024-11-11T22:16:32,179 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T22:16:32,179 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T22:16:32,179 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:16:32,182 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T22:16:32,184 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67996119, jitterRate=0.013221129775047302}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:16:32,184 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T22:16:32,185 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731363392156Writing region info on filesystem at 1731363392156Initializing all the Stores at 1731363392158 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363392158Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363392159 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363392159Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731363392159Cleaning up temporary data from old regions at 1731363392179 (+20 ms)Running coprocessor post-open hooks at 1731363392184 (+5 ms)Region opened successfully at 1731363392185 (+1 ms) 2024-11-11T22:16:32,191 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731363392090 2024-11-11T22:16:32,201 DEBUG [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T22:16:32,202 INFO [RS_OPEN_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T22:16:32,204 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:32,207 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8d82b89dd27d,36911,1731363387997, state=OPEN 2024-11-11T22:16:32,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T22:16:32,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T22:16:32,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T22:16:32,341 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T22:16:32,342 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:32,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T22:16:32,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8d82b89dd27d,36911,1731363387997 in 425 msec 2024-11-11T22:16:32,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T22:16:32,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 655 msec 2024-11-11T22:16:32,362 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T22:16:32,362 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T22:16:32,381 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:32,382 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:32,401 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:32,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58527, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:32,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7010 sec 2024-11-11T22:16:32,423 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731363392422, completionTime=-1 2024-11-11T22:16:32,425 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T22:16:32,425 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-11T22:16:32,449 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-11T22:16:32,450 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731363452450 2024-11-11T22:16:32,450 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731363512450 2024-11-11T22:16:32,450 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-11T22:16:32,452 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,41919,1731363387169-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:32,452 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,41919,1731363387169-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:32,452 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,41919,1731363387169-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:32,453 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8d82b89dd27d:41919, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:32,454 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:32,454 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:32,459 DEBUG [master/8d82b89dd27d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T22:16:32,482 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.299sec 2024-11-11T22:16:32,483 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T22:16:32,484 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T22:16:32,485 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T22:16:32,486 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T22:16:32,486 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T22:16:32,486 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,41919,1731363387169-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T22:16:32,487 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,41919,1731363387169-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T22:16:32,494 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T22:16:32,495 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T22:16:32,495 INFO [master/8d82b89dd27d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8d82b89dd27d,41919,1731363387169-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T22:16:32,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:32,524 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T22:16:32,525 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T22:16:32,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:32,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:32,539 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:32,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8ea0a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:32,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:32,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:32,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:32,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:32,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:32,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:32,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:32,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:32,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=536, ProcessCount=11, AvailableMemoryMB=1756 2024-11-11T22:16:32,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:16:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23cc1f5f 2024-11-11T22:16:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:16:32,649 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:16:32,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:16:32,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:16:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:16:32,673 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:16:32,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T22:16:32,676 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:32,678 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:16:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:16:32,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741835_1011 (size=1061) 2024-11-11T22:16:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:16:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:16:33,096 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e 2024-11-11T22:16:33,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741836_1012 (size=53) 2024-11-11T22:16:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:16:33,507 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:16:33,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3f60f99bf48ef670ca729411903c0391, disabling compactions & flushes 2024-11-11T22:16:33,508 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:33,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:33,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. after waiting 0 ms 2024-11-11T22:16:33,508 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:33,508 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:33,509 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3f60f99bf48ef670ca729411903c0391: Waiting for close lock at 1731363393508Disabling compacts and flushes for region at 1731363393508Disabling writes for close at 1731363393508Writing region close event to WAL at 1731363393508Closed at 1731363393508 2024-11-11T22:16:33,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:16:33,520 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363393513"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363393513"}]},"ts":"1731363393513"} 2024-11-11T22:16:33,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:16:33,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:16:33,529 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363393527"}]},"ts":"1731363393527"} 2024-11-11T22:16:33,533 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:16:33,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, ASSIGN}] 2024-11-11T22:16:33,538 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, ASSIGN 2024-11-11T22:16:33,539 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, ASSIGN; state=OFFLINE, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=false 2024-11-11T22:16:33,693 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3f60f99bf48ef670ca729411903c0391, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:33,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, ASSIGN because future has completed 2024-11-11T22:16:33,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:16:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:16:33,866 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:33,867 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:16:33,867 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,868 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:16:33,868 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,868 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,871 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,875 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:16:33,876 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f60f99bf48ef670ca729411903c0391 columnFamilyName A 2024-11-11T22:16:33,876 DEBUG [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:33,878 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.HStore(327): Store=3f60f99bf48ef670ca729411903c0391/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:16:33,878 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,881 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:16:33,881 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f60f99bf48ef670ca729411903c0391 columnFamilyName B 2024-11-11T22:16:33,881 DEBUG [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:33,883 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.HStore(327): Store=3f60f99bf48ef670ca729411903c0391/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:16:33,883 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,885 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:16:33,886 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f60f99bf48ef670ca729411903c0391 columnFamilyName C 2024-11-11T22:16:33,886 DEBUG [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:16:33,887 INFO [StoreOpener-3f60f99bf48ef670ca729411903c0391-1 {}] regionserver.HStore(327): Store=3f60f99bf48ef670ca729411903c0391/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:16:33,887 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:33,887 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,889 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,890 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,892 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,892 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,893 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:16:33,895 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,900 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:16:33,901 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3f60f99bf48ef670ca729411903c0391; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72994945, jitterRate=0.087709441781044}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:16:33,901 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:33,902 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3f60f99bf48ef670ca729411903c0391: Running coprocessor pre-open hook at 1731363393868Writing region info on filesystem at 1731363393868Initializing all the Stores at 1731363393870 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363393870Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363393870Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363393870Cleaning up temporary data from old regions at 1731363393892 (+22 ms)Running coprocessor post-open hooks at 1731363393901 (+9 ms)Region opened successfully at 1731363393902 (+1 ms) 2024-11-11T22:16:33,904 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., pid=6, masterSystemTime=1731363393857 2024-11-11T22:16:33,908 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:33,908 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:33,909 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3f60f99bf48ef670ca729411903c0391, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:33,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:16:33,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T22:16:33,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 in 213 msec 2024-11-11T22:16:33,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T22:16:33,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, ASSIGN in 384 msec 2024-11-11T22:16:33,926 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:16:33,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363393926"}]},"ts":"1731363393926"} 2024-11-11T22:16:33,929 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:16:33,931 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:16:33,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2670 sec 2024-11-11T22:16:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T22:16:34,844 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:34,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ede62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,853 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@455d50bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,856 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da125de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,858 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,865 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:16:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,41919,1731363387169 2024-11-11T22:16:34,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@409ffccf 2024-11-11T22:16:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:16:34,871 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:16:34,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812e936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5c6971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a084b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e500c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,885 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0b2729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,889 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6b748f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,892 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33125057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,896 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52853965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77155d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3c5a0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fdb4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdceae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63258065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,911 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347812b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58df653d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad2755a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369fd76d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85187e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,924 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355546b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,926 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d4d256, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6fa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,929 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2815dd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,932 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,933 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7994a976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a990c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:16:34,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:16:34,935 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:16:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:16:34,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:16:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4852f5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:16:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:16:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:34,937 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:16:34,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c92072, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:16:34,943 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,943 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,944 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,945 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,947 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T22:16:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,955 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:34,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:16:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:34,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:34,961 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,964 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:34,965 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,969 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:34,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:34,977 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:16:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:16:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:16:34,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:16:34,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:16:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:35,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:16:35,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:35,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:16:35,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:35,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363455152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363455158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731363395110, max timestamp=1731363395171Num uniques 859; 2024-11-11T22:16:35,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:35,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3f24f20d12724335945df725acaafda1 is 50, key is test_row_0/A:col10/1731363395028/Put/seqid=0 2024-11-11T22:16:35,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363395110, max timestamp=1731363395171Num uniques 859; 2024-11-11T22:16:35,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:35,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363395110, max timestamp=1731363395171Num uniques 859; 2024-11-11T22:16:35,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:35,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741837_1013 (size=12001) 2024-11-11T22:16:35,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3f24f20d12724335945df725acaafda1 2024-11-11T22:16:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:16:35,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:35,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:35,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363455340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363455340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363455340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/aff8cba8a6fd4134b69cf212fac1e760 is 50, key is test_row_0/B:col10/1731363395028/Put/seqid=0 2024-11-11T22:16:35,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363455352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363455353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741838_1014 (size=12001) 2024-11-11T22:16:35,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363455460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363455463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363455470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:35,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363455480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363455480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:16:35,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:35,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363455670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363455680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363455681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363455692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363455700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:35,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/aff8cba8a6fd4134b69cf212fac1e760 2024-11-11T22:16:35,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:35,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9aeba5a1914343c89aa1d31b2e9fd1c2 is 50, key is test_row_0/C:col10/1731363395028/Put/seqid=0 2024-11-11T22:16:35,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741839_1015 (size=12001) 2024-11-11T22:16:35,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:35,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:35,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363455985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:35,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363456001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363456002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363456012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363456013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:16:36,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:36,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:36,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:36,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:36,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:36,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:36,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9aeba5a1914343c89aa1d31b2e9fd1c2 2024-11-11T22:16:36,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:36,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:36,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:36,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3f24f20d12724335945df725acaafda1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3f24f20d12724335945df725acaafda1 2024-11-11T22:16:36,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3f24f20d12724335945df725acaafda1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:16:36,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/aff8cba8a6fd4134b69cf212fac1e760 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/aff8cba8a6fd4134b69cf212fac1e760 2024-11-11T22:16:36,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/aff8cba8a6fd4134b69cf212fac1e760, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:16:36,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9aeba5a1914343c89aa1d31b2e9fd1c2 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9aeba5a1914343c89aa1d31b2e9fd1c2 2024-11-11T22:16:36,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9aeba5a1914343c89aa1d31b2e9fd1c2, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:16:36,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.53 KB/121370 for 3f60f99bf48ef670ca729411903c0391 in 1361ms, sequenceid=13, compaction requested=false 2024-11-11T22:16:36,395 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T22:16:36,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:36,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T22:16:36,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:36,454 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=118.53 KB heapSize=335.38 KB 2024-11-11T22:16:36,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:36,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:36,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/ffe5de3d94fa4a27b581a8364b57aeb4 is 50, key is test_row_0/A:col10/1731363395347/Put/seqid=0 2024-11-11T22:16:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:36,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:36,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741840_1016 (size=12001) 2024-11-11T22:16:36,560 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/ffe5de3d94fa4a27b581a8364b57aeb4 2024-11-11T22:16:36,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/0b08798ce9ad42eca345cabcc0be6d55 is 50, key is test_row_0/B:col10/1731363395347/Put/seqid=0 2024-11-11T22:16:36,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363456595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363456604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741841_1017 (size=12001) 2024-11-11T22:16:36,646 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/0b08798ce9ad42eca345cabcc0be6d55 2024-11-11T22:16:36,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363456633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/6dff705f4509487e8c79e655153f39cc is 50, key is test_row_0/C:col10/1731363395347/Put/seqid=0 2024-11-11T22:16:36,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363456642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363456642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741842_1018 (size=12001) 2024-11-11T22:16:36,711 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/6dff705f4509487e8c79e655153f39cc 2024-11-11T22:16:36,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/ffe5de3d94fa4a27b581a8364b57aeb4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ffe5de3d94fa4a27b581a8364b57aeb4 2024-11-11T22:16:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363456752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363456752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,777 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ffe5de3d94fa4a27b581a8364b57aeb4, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T22:16:36,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/0b08798ce9ad42eca345cabcc0be6d55 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/0b08798ce9ad42eca345cabcc0be6d55 2024-11-11T22:16:36,798 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/0b08798ce9ad42eca345cabcc0be6d55, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T22:16:36,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/6dff705f4509487e8c79e655153f39cc as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6dff705f4509487e8c79e655153f39cc 2024-11-11T22:16:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363456781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363456791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363456792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:36,821 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6dff705f4509487e8c79e655153f39cc, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T22:16:36,829 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~118.53 KB/121370, heapSize ~335.33 KB/343376, currentSize=67.09 KB/68700 for 3f60f99bf48ef670ca729411903c0391 in 375ms, sequenceid=47, compaction requested=false 2024-11-11T22:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T22:16:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T22:16:36,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T22:16:36,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-11T22:16:36,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9000 sec 2024-11-11T22:16:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:36,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T22:16:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:36,991 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T22:16:37,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f7c0dcf408ef4527964b8b00ccaae1f0 is 50, key is test_row_0/A:col10/1731363396981/Put/seqid=0 2024-11-11T22:16:37,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741843_1019 (size=14341) 2024-11-11T22:16:37,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f7c0dcf408ef4527964b8b00ccaae1f0 2024-11-11T22:16:37,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/2429849c5016433db1337668fa2eacb5 is 50, key is test_row_0/B:col10/1731363396981/Put/seqid=0 2024-11-11T22:16:37,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741844_1020 (size=12001) 2024-11-11T22:16:37,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/2429849c5016433db1337668fa2eacb5 2024-11-11T22:16:37,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363457052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363457063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363457064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363457065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363457067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T22:16:37,113 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:37,114 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T22:16:37,118 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:16:37,120 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:37,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:37,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/606820d66436480e8f574350c09b19d1 is 50, key is test_row_0/C:col10/1731363396981/Put/seqid=0 2024-11-11T22:16:37,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741845_1021 (size=12001) 2024-11-11T22:16:37,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/606820d66436480e8f574350c09b19d1 2024-11-11T22:16:37,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363457191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363457191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363457192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363457192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f7c0dcf408ef4527964b8b00ccaae1f0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f7c0dcf408ef4527964b8b00ccaae1f0 2024-11-11T22:16:37,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363457190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f7c0dcf408ef4527964b8b00ccaae1f0, entries=200, sequenceid=62, filesize=14.0 K 2024-11-11T22:16:37,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/2429849c5016433db1337668fa2eacb5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2429849c5016433db1337668fa2eacb5 2024-11-11T22:16:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:16:37,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2429849c5016433db1337668fa2eacb5, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T22:16:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/606820d66436480e8f574350c09b19d1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/606820d66436480e8f574350c09b19d1 2024-11-11T22:16:37,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/606820d66436480e8f574350c09b19d1, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T22:16:37,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 3f60f99bf48ef670ca729411903c0391 in 282ms, sequenceid=62, compaction requested=true 2024-11-11T22:16:37,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:37,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:37,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T22:16:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:37,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:37,275 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:16:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:37,276 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:37,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:37,276 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:37,281 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:37,283 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:37,284 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:37,285 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/aff8cba8a6fd4134b69cf212fac1e760, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/0b08798ce9ad42eca345cabcc0be6d55, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2429849c5016433db1337668fa2eacb5] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.2 K 2024-11-11T22:16:37,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:37,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:37,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:37,288 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting aff8cba8a6fd4134b69cf212fac1e760, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363395000 2024-11-11T22:16:37,289 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:37,289 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b08798ce9ad42eca345cabcc0be6d55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731363395242 2024-11-11T22:16:37,290 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2429849c5016433db1337668fa2eacb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363396631 2024-11-11T22:16:37,290 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/6992680497964146ab63fc2e78feb1d4 is 50, key is test_row_0/A:col10/1731363397036/Put/seqid=0 2024-11-11T22:16:37,295 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:37,296 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3f24f20d12724335945df725acaafda1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ffe5de3d94fa4a27b581a8364b57aeb4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f7c0dcf408ef4527964b8b00ccaae1f0] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=37.4 K 2024-11-11T22:16:37,299 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f24f20d12724335945df725acaafda1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363395000 2024-11-11T22:16:37,302 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffe5de3d94fa4a27b581a8364b57aeb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731363395242 2024-11-11T22:16:37,304 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c0dcf408ef4527964b8b00ccaae1f0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363396585 2024-11-11T22:16:37,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741846_1022 (size=12001) 2024-11-11T22:16:37,343 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/6992680497964146ab63fc2e78feb1d4 2024-11-11T22:16:37,377 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#10 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:37,381 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/3d14e2756f0646d8816b93a57c775b42 is 50, key is test_row_0/B:col10/1731363396981/Put/seqid=0 2024-11-11T22:16:37,382 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:37,384 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/30bdd39a409442ad8e40ec8930023dfe is 50, key is test_row_0/A:col10/1731363396981/Put/seqid=0 2024-11-11T22:16:37,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/500a0726ec63496d9207d6ff2ba0154c is 50, key is test_row_0/B:col10/1731363397036/Put/seqid=0 2024-11-11T22:16:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:37,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:37,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741847_1023 (size=12104) 2024-11-11T22:16:37,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741848_1024 (size=12104) 2024-11-11T22:16:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:16:37,444 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/3d14e2756f0646d8816b93a57c775b42 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d14e2756f0646d8816b93a57c775b42 2024-11-11T22:16:37,451 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/30bdd39a409442ad8e40ec8930023dfe as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/30bdd39a409442ad8e40ec8930023dfe 2024-11-11T22:16:37,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741849_1025 (size=12001) 2024-11-11T22:16:37,476 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into 3d14e2756f0646d8816b93a57c775b42(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:37,476 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:37,478 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into 30bdd39a409442ad8e40ec8930023dfe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:37,478 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:37,480 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=13, startTime=1731363397275; duration=0sec 2024-11-11T22:16:37,481 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:37,482 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:37,482 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:37,480 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=13, startTime=1731363397270; duration=0sec 2024-11-11T22:16:37,484 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:37,484 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:37,484 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:37,485 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:37,485 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:37,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,485 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9aeba5a1914343c89aa1d31b2e9fd1c2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6dff705f4509487e8c79e655153f39cc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/606820d66436480e8f574350c09b19d1] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.2 K 2024-11-11T22:16:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363457460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363457462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,487 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aeba5a1914343c89aa1d31b2e9fd1c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363395000 2024-11-11T22:16:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363457463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363457471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,493 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dff705f4509487e8c79e655153f39cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731363395242 2024-11-11T22:16:37,494 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 606820d66436480e8f574350c09b19d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363396631 2024-11-11T22:16:37,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363457485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,521 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:37,525 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/61a24b7b34e945e0aa6ca9282e15db66 is 50, key is test_row_0/C:col10/1731363396981/Put/seqid=0 2024-11-11T22:16:37,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741850_1026 (size=12104) 2024-11-11T22:16:37,573 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/61a24b7b34e945e0aa6ca9282e15db66 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/61a24b7b34e945e0aa6ca9282e15db66 2024-11-11T22:16:37,591 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into 61a24b7b34e945e0aa6ca9282e15db66(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:37,592 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:37,592 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=13, startTime=1731363397286; duration=0sec 2024-11-11T22:16:37,592 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:37,592 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:37,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363457590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363457590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363457600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363457600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363457609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:16:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:16:37,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363457805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363457822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363457821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363457825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363457825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:37,858 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T22:16:37,858 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T22:16:37,861 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T22:16:37,861 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T22:16:37,862 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T22:16:37,862 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T22:16:37,864 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T22:16:37,864 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T22:16:37,870 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/500a0726ec63496d9207d6ff2ba0154c 2024-11-11T22:16:37,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/d753f4548f5d4a31bae4fe29146ac04a is 50, key is test_row_0/C:col10/1731363397036/Put/seqid=0 2024-11-11T22:16:37,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741851_1027 (size=12001) 2024-11-11T22:16:37,965 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/d753f4548f5d4a31bae4fe29146ac04a 2024-11-11T22:16:37,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/6992680497964146ab63fc2e78feb1d4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/6992680497964146ab63fc2e78feb1d4 2024-11-11T22:16:38,015 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/6992680497964146ab63fc2e78feb1d4, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:16:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/500a0726ec63496d9207d6ff2ba0154c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/500a0726ec63496d9207d6ff2ba0154c 2024-11-11T22:16:38,030 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/500a0726ec63496d9207d6ff2ba0154c, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:16:38,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/d753f4548f5d4a31bae4fe29146ac04a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d753f4548f5d4a31bae4fe29146ac04a 2024-11-11T22:16:38,063 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d753f4548f5d4a31bae4fe29146ac04a, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:16:38,065 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 3f60f99bf48ef670ca729411903c0391 in 789ms, sequenceid=79, compaction requested=false 2024-11-11T22:16:38,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:38,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:38,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T22:16:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T22:16:38,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T22:16:38,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 948 msec 2024-11-11T22:16:38,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 961 msec 2024-11-11T22:16:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:38,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:16:38,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:38,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/5e8558d86e764ca69aaed741adb9f4d0 is 50, key is test_row_0/A:col10/1731363397451/Put/seqid=0 2024-11-11T22:16:38,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741852_1028 (size=12001) 2024-11-11T22:16:38,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/5e8558d86e764ca69aaed741adb9f4d0 2024-11-11T22:16:38,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363458201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363458200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363458201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363458208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363458204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/52234c903baf447e9bde7d15ab6337b0 is 50, key is test_row_0/B:col10/1731363397451/Put/seqid=0 2024-11-11T22:16:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T22:16:38,260 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:38,260 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T22:16:38,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:16:38,269 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:38,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:38,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741853_1029 (size=12001) 2024-11-11T22:16:38,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/52234c903baf447e9bde7d15ab6337b0 2024-11-11T22:16:38,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/1a0566565cf44598b6727fefb6f942ad is 50, key is test_row_0/C:col10/1731363397451/Put/seqid=0 2024-11-11T22:16:38,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741854_1030 (size=12001) 2024-11-11T22:16:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363458330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/1a0566565cf44598b6727fefb6f942ad 2024-11-11T22:16:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363458330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363458331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363458331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363458339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/5e8558d86e764ca69aaed741adb9f4d0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e8558d86e764ca69aaed741adb9f4d0 2024-11-11T22:16:38,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e8558d86e764ca69aaed741adb9f4d0, entries=150, sequenceid=99, filesize=11.7 K 2024-11-11T22:16:38,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/52234c903baf447e9bde7d15ab6337b0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/52234c903baf447e9bde7d15ab6337b0 2024-11-11T22:16:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:16:38,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/52234c903baf447e9bde7d15ab6337b0, entries=150, sequenceid=99, filesize=11.7 K 2024-11-11T22:16:38,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/1a0566565cf44598b6727fefb6f942ad as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/1a0566565cf44598b6727fefb6f942ad 2024-11-11T22:16:38,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/1a0566565cf44598b6727fefb6f942ad, entries=150, sequenceid=99, filesize=11.7 K 2024-11-11T22:16:38,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 3f60f99bf48ef670ca729411903c0391 in 271ms, sequenceid=99, compaction requested=true 2024-11-11T22:16:38,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:38,411 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:38,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:38,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:38,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:38,413 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:38,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:38,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:38,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:38,414 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:38,414 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:38,414 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:38,415 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/30bdd39a409442ad8e40ec8930023dfe, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/6992680497964146ab63fc2e78feb1d4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e8558d86e764ca69aaed741adb9f4d0] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.3 K 2024-11-11T22:16:38,416 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30bdd39a409442ad8e40ec8930023dfe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363396631 2024-11-11T22:16:38,416 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:38,416 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:38,417 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:38,417 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d14e2756f0646d8816b93a57c775b42, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/500a0726ec63496d9207d6ff2ba0154c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/52234c903baf447e9bde7d15ab6337b0] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.3 K 2024-11-11T22:16:38,419 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6992680497964146ab63fc2e78feb1d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363397036 2024-11-11T22:16:38,419 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d14e2756f0646d8816b93a57c775b42, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363396631 2024-11-11T22:16:38,420 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 500a0726ec63496d9207d6ff2ba0154c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363397036 2024-11-11T22:16:38,420 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8558d86e764ca69aaed741adb9f4d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731363397451 2024-11-11T22:16:38,422 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 52234c903baf447e9bde7d15ab6337b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731363397451 2024-11-11T22:16:38,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T22:16:38,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:38,427 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T22:16:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:38,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:38,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:38,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/12fd55b8df4647b9b4ef547119f2dbd2 is 50, key is test_row_0/A:col10/1731363398198/Put/seqid=0 2024-11-11T22:16:38,450 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:38,451 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/703d29192de14a098a0ee14c3f17cdd0 is 50, key is test_row_0/A:col10/1731363397451/Put/seqid=0 2024-11-11T22:16:38,456 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#20 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:38,458 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/4e47121de60c41149f456bb9a327a750 is 50, key is test_row_0/B:col10/1731363397451/Put/seqid=0 2024-11-11T22:16:38,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741856_1032 (size=12207) 2024-11-11T22:16:38,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741855_1031 (size=12001) 2024-11-11T22:16:38,488 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/12fd55b8df4647b9b4ef547119f2dbd2 2024-11-11T22:16:38,505 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/703d29192de14a098a0ee14c3f17cdd0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/703d29192de14a098a0ee14c3f17cdd0 2024-11-11T22:16:38,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741857_1033 (size=12207) 2024-11-11T22:16:38,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/3737b4fca4f94f50b76c1337fa0f685e is 50, key is test_row_0/B:col10/1731363398198/Put/seqid=0 2024-11-11T22:16:38,524 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into 703d29192de14a098a0ee14c3f17cdd0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:38,524 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:38,525 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=13, startTime=1731363398411; duration=0sec 2024-11-11T22:16:38,525 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:38,525 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:38,525 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:38,531 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:38,531 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:38,531 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:38,531 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/4e47121de60c41149f456bb9a327a750 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/4e47121de60c41149f456bb9a327a750 2024-11-11T22:16:38,531 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/61a24b7b34e945e0aa6ca9282e15db66, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d753f4548f5d4a31bae4fe29146ac04a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/1a0566565cf44598b6727fefb6f942ad] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.3 K 2024-11-11T22:16:38,532 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a24b7b34e945e0aa6ca9282e15db66, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363396631 2024-11-11T22:16:38,533 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d753f4548f5d4a31bae4fe29146ac04a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363397036 2024-11-11T22:16:38,534 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a0566565cf44598b6727fefb6f942ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731363397451 2024-11-11T22:16:38,543 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into 4e47121de60c41149f456bb9a327a750(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:38,543 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:38,543 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=13, startTime=1731363398413; duration=0sec 2024-11-11T22:16:38,544 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:38,544 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:38,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:38,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741858_1034 (size=12001) 2024-11-11T22:16:38,557 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/3737b4fca4f94f50b76c1337fa0f685e 2024-11-11T22:16:38,567 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:38,568 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/e1f7b7c14038432f89b252a01201c965 is 50, key is test_row_0/C:col10/1731363397451/Put/seqid=0 2024-11-11T22:16:38,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:16:38,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/973073218b0a4da0aeef97f022d78ce4 is 50, key is test_row_0/C:col10/1731363398198/Put/seqid=0 2024-11-11T22:16:38,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363458589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363458593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363458597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363458603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363458603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741859_1035 (size=12207) 2024-11-11T22:16:38,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741860_1036 (size=12001) 2024-11-11T22:16:38,633 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/973073218b0a4da0aeef97f022d78ce4 2024-11-11T22:16:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/12fd55b8df4647b9b4ef547119f2dbd2 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/12fd55b8df4647b9b4ef547119f2dbd2 2024-11-11T22:16:38,662 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/12fd55b8df4647b9b4ef547119f2dbd2, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T22:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/3737b4fca4f94f50b76c1337fa0f685e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3737b4fca4f94f50b76c1337fa0f685e 2024-11-11T22:16:38,679 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3737b4fca4f94f50b76c1337fa0f685e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T22:16:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/973073218b0a4da0aeef97f022d78ce4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/973073218b0a4da0aeef97f022d78ce4 2024-11-11T22:16:38,695 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/973073218b0a4da0aeef97f022d78ce4, entries=150, sequenceid=115, filesize=11.7 K 2024-11-11T22:16:38,698 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 3f60f99bf48ef670ca729411903c0391 in 271ms, sequenceid=115, compaction requested=false 2024-11-11T22:16:38,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:38,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T22:16:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T22:16:38,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T22:16:38,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 431 msec 2024-11-11T22:16:38,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T22:16:38,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 447 msec 2024-11-11T22:16:38,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:38,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:38,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:38,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:38,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3a145e8c1b054f24846f9f95b890874a is 50, key is test_row_0/A:col10/1731363398599/Put/seqid=0 2024-11-11T22:16:38,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741861_1037 (size=14491) 2024-11-11T22:16:38,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3a145e8c1b054f24846f9f95b890874a 2024-11-11T22:16:38,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363458748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363458757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363458763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363458770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363458773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/dcdf05b2be6e465aa8016d7ce6efa136 is 50, key is test_row_0/B:col10/1731363398599/Put/seqid=0 2024-11-11T22:16:38,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741862_1038 (size=12101) 2024-11-11T22:16:38,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363458879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363458879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T22:16:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,891 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:38,891 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T22:16:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363458891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:16:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363458892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363458893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:38,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:38,902 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:38,903 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:38,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:16:39,048 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/e1f7b7c14038432f89b252a01201c965 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e1f7b7c14038432f89b252a01201c965 2024-11-11T22:16:39,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T22:16:39,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:39,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:39,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:39,064 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into e1f7b7c14038432f89b252a01201c965(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:39,064 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:39,064 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=13, startTime=1731363398413; duration=0sec 2024-11-11T22:16:39,065 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:39,065 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:39,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363459089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363459102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363459109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363459115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363459114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:16:39,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T22:16:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:39,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:39,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/dcdf05b2be6e465aa8016d7ce6efa136 2024-11-11T22:16:39,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/a31a8745da9b4bfea895763370184750 is 50, key is test_row_0/C:col10/1731363398599/Put/seqid=0 2024-11-11T22:16:39,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741863_1039 (size=12101) 2024-11-11T22:16:39,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/a31a8745da9b4bfea895763370184750 2024-11-11T22:16:39,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3a145e8c1b054f24846f9f95b890874a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3a145e8c1b054f24846f9f95b890874a 2024-11-11T22:16:39,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3a145e8c1b054f24846f9f95b890874a, entries=200, sequenceid=135, filesize=14.2 K 2024-11-11T22:16:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/dcdf05b2be6e465aa8016d7ce6efa136 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcdf05b2be6e465aa8016d7ce6efa136 2024-11-11T22:16:39,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcdf05b2be6e465aa8016d7ce6efa136, entries=150, sequenceid=135, filesize=11.8 K 2024-11-11T22:16:39,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/a31a8745da9b4bfea895763370184750 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a31a8745da9b4bfea895763370184750 2024-11-11T22:16:39,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a31a8745da9b4bfea895763370184750, entries=150, sequenceid=135, filesize=11.8 K 2024-11-11T22:16:39,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 3f60f99bf48ef670ca729411903c0391 in 596ms, sequenceid=135, compaction requested=true 2024-11-11T22:16:39,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:39,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:39,309 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:39,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:39,309 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:39,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:39,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:39,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:39,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:39,312 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:39,312 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:39,313 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:39,313 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:39,313 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,313 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,313 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/703d29192de14a098a0ee14c3f17cdd0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/12fd55b8df4647b9b4ef547119f2dbd2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3a145e8c1b054f24846f9f95b890874a] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=37.8 K 2024-11-11T22:16:39,313 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/4e47121de60c41149f456bb9a327a750, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3737b4fca4f94f50b76c1337fa0f685e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcdf05b2be6e465aa8016d7ce6efa136] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.5 K 2024-11-11T22:16:39,317 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 703d29192de14a098a0ee14c3f17cdd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731363397451 2024-11-11T22:16:39,317 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e47121de60c41149f456bb9a327a750, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731363397451 2024-11-11T22:16:39,318 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12fd55b8df4647b9b4ef547119f2dbd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731363398183 2024-11-11T22:16:39,318 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3737b4fca4f94f50b76c1337fa0f685e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731363398183 2024-11-11T22:16:39,318 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a145e8c1b054f24846f9f95b890874a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731363398576 2024-11-11T22:16:39,319 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting dcdf05b2be6e465aa8016d7ce6efa136, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731363398576 2024-11-11T22:16:39,353 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:39,354 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/495bd65b6158492b8ce8af4e2821d92f is 50, key is test_row_0/A:col10/1731363398599/Put/seqid=0 2024-11-11T22:16:39,363 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:39,364 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/a652a83b161a487c86697754268b494f is 50, key is test_row_0/B:col10/1731363398599/Put/seqid=0 2024-11-11T22:16:39,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T22:16:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,372 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T22:16:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:39,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/dc0d51d103804e39af5dc0fc220ae846 is 50, key is test_row_0/A:col10/1731363398766/Put/seqid=0 2024-11-11T22:16:39,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741864_1040 (size=12409) 2024-11-11T22:16:39,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741865_1041 (size=12409) 2024-11-11T22:16:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:39,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:39,426 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/a652a83b161a487c86697754268b494f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/a652a83b161a487c86697754268b494f 2024-11-11T22:16:39,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741866_1042 (size=12151) 2024-11-11T22:16:39,428 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/dc0d51d103804e39af5dc0fc220ae846 2024-11-11T22:16:39,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/bc25b6d5dda743f6ac3c50715ee95801 is 50, key is test_row_0/B:col10/1731363398766/Put/seqid=0 2024-11-11T22:16:39,475 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into a652a83b161a487c86697754268b494f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:39,475 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:39,475 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=13, startTime=1731363399309; duration=0sec 2024-11-11T22:16:39,476 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:39,476 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:39,476 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:39,480 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:39,480 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:39,480 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,480 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e1f7b7c14038432f89b252a01201c965, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/973073218b0a4da0aeef97f022d78ce4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a31a8745da9b4bfea895763370184750] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.5 K 2024-11-11T22:16:39,481 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e1f7b7c14038432f89b252a01201c965, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731363397451 2024-11-11T22:16:39,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741867_1043 (size=12151) 2024-11-11T22:16:39,483 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 973073218b0a4da0aeef97f022d78ce4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731363398183 2024-11-11T22:16:39,483 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a31a8745da9b4bfea895763370184750, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731363398576 2024-11-11T22:16:39,485 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/bc25b6d5dda743f6ac3c50715ee95801 2024-11-11T22:16:39,508 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:39,509 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/7f2eb2cdb16b4d40a8301e9c4d3ab9b5 is 50, key is test_row_0/C:col10/1731363398599/Put/seqid=0 2024-11-11T22:16:39,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/d632aa6ba333411ab1b3cee554888eb4 is 50, key is test_row_0/C:col10/1731363398766/Put/seqid=0 2024-11-11T22:16:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:16:39,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363459518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363459519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363459519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363459529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363459531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741868_1044 (size=12409) 2024-11-11T22:16:39,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741869_1045 (size=12151) 2024-11-11T22:16:39,551 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/d632aa6ba333411ab1b3cee554888eb4 2024-11-11T22:16:39,564 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/7f2eb2cdb16b4d40a8301e9c4d3ab9b5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7f2eb2cdb16b4d40a8301e9c4d3ab9b5 2024-11-11T22:16:39,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/dc0d51d103804e39af5dc0fc220ae846 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/dc0d51d103804e39af5dc0fc220ae846 2024-11-11T22:16:39,585 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/dc0d51d103804e39af5dc0fc220ae846, entries=150, sequenceid=150, filesize=11.9 K 2024-11-11T22:16:39,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/bc25b6d5dda743f6ac3c50715ee95801 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/bc25b6d5dda743f6ac3c50715ee95801 2024-11-11T22:16:39,591 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into 7f2eb2cdb16b4d40a8301e9c4d3ab9b5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:39,591 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:39,591 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=13, startTime=1731363399309; duration=0sec 2024-11-11T22:16:39,592 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:39,592 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:39,606 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/bc25b6d5dda743f6ac3c50715ee95801, entries=150, sequenceid=150, filesize=11.9 K 2024-11-11T22:16:39,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/d632aa6ba333411ab1b3cee554888eb4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d632aa6ba333411ab1b3cee554888eb4 2024-11-11T22:16:39,624 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d632aa6ba333411ab1b3cee554888eb4, entries=150, sequenceid=150, filesize=11.9 K 2024-11-11T22:16:39,626 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3f60f99bf48ef670ca729411903c0391 in 254ms, sequenceid=150, compaction requested=false 2024-11-11T22:16:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:39,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T22:16:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T22:16:39,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T22:16:39,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 726 msec 2024-11-11T22:16:39,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 743 msec 2024-11-11T22:16:39,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:16:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:39,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363399460, max timestamp=1731363399649Num uniques 859; 2024-11-11T22:16:39,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363399460, max timestamp=1731363399649Num uniques 859; 2024-11-11T22:16:39,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:39,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:39,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:39,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:39,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:39,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:39,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:39,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:39,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/c0034c46457445cea6ecefbbfb0246ac is 50, key is test_row_0/A:col10/1731363399647/Put/seqid=0 2024-11-11T22:16:39,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741870_1046 (size=12151) 2024-11-11T22:16:39,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/c0034c46457445cea6ecefbbfb0246ac 2024-11-11T22:16:39,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/d67b5dd6c5874786abfaf58631fa351c is 50, key is test_row_0/B:col10/1731363399647/Put/seqid=0 2024-11-11T22:16:39,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363399694, max timestamp=1731363399725Num uniques 859; 2024-11-11T22:16:39,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:39,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363399694, max timestamp=1731363399724Num uniques 859; 2024-11-11T22:16:39,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:39,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363399694, max timestamp=1731363399725Num uniques 859; 2024-11-11T22:16:39,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:39,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741871_1047 (size=12151) 2024-11-11T22:16:39,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363459791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363459792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363459791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363459794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363459795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,811 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/495bd65b6158492b8ce8af4e2821d92f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/495bd65b6158492b8ce8af4e2821d92f 2024-11-11T22:16:39,823 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into 495bd65b6158492b8ce8af4e2821d92f(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:39,823 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:39,824 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=13, startTime=1731363399309; duration=0sec 2024-11-11T22:16:39,824 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:39,824 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:39,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363459910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363459910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363459910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363459910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:39,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363459910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T22:16:40,030 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:40,030 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T22:16:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:16:40,035 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:40,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:40,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:40,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363460120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363460120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363460121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363460121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363460130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:16:40,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/d67b5dd6c5874786abfaf58631fa351c 2024-11-11T22:16:40,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T22:16:40,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:40,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:40,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:40,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:40,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:40,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/0f2f4ab68e2a4552b9f6328628275a61 is 50, key is test_row_0/C:col10/1731363399647/Put/seqid=0 2024-11-11T22:16:40,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741872_1048 (size=12151) 2024-11-11T22:16:40,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/0f2f4ab68e2a4552b9f6328628275a61 2024-11-11T22:16:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/c0034c46457445cea6ecefbbfb0246ac as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c0034c46457445cea6ecefbbfb0246ac 2024-11-11T22:16:40,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c0034c46457445cea6ecefbbfb0246ac, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T22:16:40,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/d67b5dd6c5874786abfaf58631fa351c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/d67b5dd6c5874786abfaf58631fa351c 2024-11-11T22:16:40,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/d67b5dd6c5874786abfaf58631fa351c, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T22:16:40,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/0f2f4ab68e2a4552b9f6328628275a61 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/0f2f4ab68e2a4552b9f6328628275a61 2024-11-11T22:16:40,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/0f2f4ab68e2a4552b9f6328628275a61, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T22:16:40,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 3f60f99bf48ef670ca729411903c0391 in 663ms, sequenceid=173, compaction requested=true 2024-11-11T22:16:40,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:40,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:40,314 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:40,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:40,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:40,314 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:40,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:40,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:40,316 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:40,316 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:40,316 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:40,317 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/495bd65b6158492b8ce8af4e2821d92f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/dc0d51d103804e39af5dc0fc220ae846, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c0034c46457445cea6ecefbbfb0246ac] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.9 K 2024-11-11T22:16:40,317 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:40,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:40,317 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:40,317 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:40,317 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/a652a83b161a487c86697754268b494f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/bc25b6d5dda743f6ac3c50715ee95801, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/d67b5dd6c5874786abfaf58631fa351c] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.9 K 2024-11-11T22:16:40,318 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 495bd65b6158492b8ce8af4e2821d92f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731363398576 2024-11-11T22:16:40,318 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a652a83b161a487c86697754268b494f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731363398576 2024-11-11T22:16:40,320 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bc25b6d5dda743f6ac3c50715ee95801, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731363398740 2024-11-11T22:16:40,321 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc0d51d103804e39af5dc0fc220ae846, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731363398740 2024-11-11T22:16:40,322 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d67b5dd6c5874786abfaf58631fa351c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363399645 2024-11-11T22:16:40,323 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0034c46457445cea6ecefbbfb0246ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363399645 2024-11-11T22:16:40,345 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#36 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:40,345 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/80ed7e4bae454b1cb4fcd6cb8edd1688 is 50, key is test_row_0/A:col10/1731363399647/Put/seqid=0 2024-11-11T22:16:40,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T22:16:40,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:40,349 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T22:16:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:16:40,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:40,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:40,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:40,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:40,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:40,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:40,362 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:40,363 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/dcfa020f1dc941c8ae46e7262629a2c7 is 50, key is test_row_0/B:col10/1731363399647/Put/seqid=0 2024-11-11T22:16:40,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f25270bd51e848d4b9376a4442f9da70 is 50, key is test_row_0/A:col10/1731363399789/Put/seqid=0 2024-11-11T22:16:40,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741873_1049 (size=12561) 2024-11-11T22:16:40,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741874_1050 (size=12561) 2024-11-11T22:16:40,410 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/80ed7e4bae454b1cb4fcd6cb8edd1688 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/80ed7e4bae454b1cb4fcd6cb8edd1688 2024-11-11T22:16:40,412 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/dcfa020f1dc941c8ae46e7262629a2c7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcfa020f1dc941c8ae46e7262629a2c7 2024-11-11T22:16:40,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741875_1051 (size=12151) 2024-11-11T22:16:40,425 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into 80ed7e4bae454b1cb4fcd6cb8edd1688(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:40,426 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f25270bd51e848d4b9376a4442f9da70 2024-11-11T22:16:40,426 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:40,426 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=13, startTime=1731363400314; duration=0sec 2024-11-11T22:16:40,426 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:40,426 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:40,426 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:40,430 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:40,430 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:40,430 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:40,430 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7f2eb2cdb16b4d40a8301e9c4d3ab9b5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d632aa6ba333411ab1b3cee554888eb4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/0f2f4ab68e2a4552b9f6328628275a61] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=35.9 K 2024-11-11T22:16:40,431 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2eb2cdb16b4d40a8301e9c4d3ab9b5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731363398576 2024-11-11T22:16:40,432 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d632aa6ba333411ab1b3cee554888eb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731363398740 2024-11-11T22:16:40,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:40,433 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f2f4ab68e2a4552b9f6328628275a61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363399645 2024-11-11T22:16:40,437 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into dcfa020f1dc941c8ae46e7262629a2c7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:40,438 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:40,442 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=13, startTime=1731363400314; duration=0sec 2024-11-11T22:16:40,443 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:40,443 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:40,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/8a6860f2169a4c07b3cebdc7cca8d614 is 50, key is test_row_0/B:col10/1731363399789/Put/seqid=0 2024-11-11T22:16:40,459 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:40,460 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/38ca90ee19a948ef92688cc96423e441 is 50, key is test_row_0/C:col10/1731363399647/Put/seqid=0 2024-11-11T22:16:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363460446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363460449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363460452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363460463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363460463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741876_1052 (size=12151) 2024-11-11T22:16:40,508 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/8a6860f2169a4c07b3cebdc7cca8d614 2024-11-11T22:16:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/91dd2d5a0939434a8e2804c413aac3f5 is 50, key is test_row_0/C:col10/1731363399789/Put/seqid=0 2024-11-11T22:16:40,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741877_1053 (size=12561) 2024-11-11T22:16:40,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741878_1054 (size=12151) 2024-11-11T22:16:40,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363460569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363460569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363460570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,576 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/38ca90ee19a948ef92688cc96423e441 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/38ca90ee19a948ef92688cc96423e441 2024-11-11T22:16:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363460579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363460579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,605 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into 38ca90ee19a948ef92688cc96423e441(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:40,605 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:40,605 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=13, startTime=1731363400315; duration=0sec 2024-11-11T22:16:40,606 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:40,606 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:16:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363460780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363460781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363460781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363460790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363460790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:40,962 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/91dd2d5a0939434a8e2804c413aac3f5 2024-11-11T22:16:40,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f25270bd51e848d4b9376a4442f9da70 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f25270bd51e848d4b9376a4442f9da70 2024-11-11T22:16:40,984 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f25270bd51e848d4b9376a4442f9da70, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T22:16:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/8a6860f2169a4c07b3cebdc7cca8d614 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/8a6860f2169a4c07b3cebdc7cca8d614 2024-11-11T22:16:41,000 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/8a6860f2169a4c07b3cebdc7cca8d614, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T22:16:41,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/91dd2d5a0939434a8e2804c413aac3f5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/91dd2d5a0939434a8e2804c413aac3f5 2024-11-11T22:16:41,016 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/91dd2d5a0939434a8e2804c413aac3f5, entries=150, sequenceid=210, filesize=11.9 K 2024-11-11T22:16:41,018 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 3f60f99bf48ef670ca729411903c0391 in 669ms, sequenceid=210, compaction requested=false 2024-11-11T22:16:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T22:16:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T22:16:41,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T22:16:41,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 984 msec 2024-11-11T22:16:41,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 998 msec 2024-11-11T22:16:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:41,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:16:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:41,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:41,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/b67502a2d1304a2f911cb1b285436411 is 50, key is test_row_0/A:col10/1731363400442/Put/seqid=0 2024-11-11T22:16:41,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741879_1055 (size=12151) 2024-11-11T22:16:41,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/b67502a2d1304a2f911cb1b285436411 2024-11-11T22:16:41,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363461134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363461135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363461138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363461139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363461140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/b0df686c180242f0b92df8b3c7f72e46 is 50, key is test_row_0/B:col10/1731363400442/Put/seqid=0 2024-11-11T22:16:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741880_1056 (size=12151) 2024-11-11T22:16:41,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/b0df686c180242f0b92df8b3c7f72e46 2024-11-11T22:16:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T22:16:41,170 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:41,170 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T22:16:41,174 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:41,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:16:41,178 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:41,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:41,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9a0b3ec157624085bf92279f8de8e4e8 is 50, key is test_row_0/C:col10/1731363400442/Put/seqid=0 2024-11-11T22:16:41,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741881_1057 (size=12151) 2024-11-11T22:16:41,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363461249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363461250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363461250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363461250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363461250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:16:41,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T22:16:41,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:41,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363461459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363461460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363461460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363461460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363461461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:16:41,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T22:16:41,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:41,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:41,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9a0b3ec157624085bf92279f8de8e4e8 2024-11-11T22:16:41,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/b67502a2d1304a2f911cb1b285436411 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b67502a2d1304a2f911cb1b285436411 2024-11-11T22:16:41,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b67502a2d1304a2f911cb1b285436411, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T22:16:41,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/b0df686c180242f0b92df8b3c7f72e46 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b0df686c180242f0b92df8b3c7f72e46 2024-11-11T22:16:41,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b0df686c180242f0b92df8b3c7f72e46, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T22:16:41,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9a0b3ec157624085bf92279f8de8e4e8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9a0b3ec157624085bf92279f8de8e4e8 2024-11-11T22:16:41,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9a0b3ec157624085bf92279f8de8e4e8, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T22:16:41,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 3f60f99bf48ef670ca729411903c0391 in 553ms, sequenceid=226, compaction requested=true 2024-11-11T22:16:41,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:41,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:41,645 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:41,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:41,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:41,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:41,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:41,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:41,646 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:41,648 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:41,648 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:41,648 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:41,648 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:41,648 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,648 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,648 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcfa020f1dc941c8ae46e7262629a2c7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/8a6860f2169a4c07b3cebdc7cca8d614, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b0df686c180242f0b92df8b3c7f72e46] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=36.0 K 2024-11-11T22:16:41,648 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/80ed7e4bae454b1cb4fcd6cb8edd1688, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f25270bd51e848d4b9376a4442f9da70, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b67502a2d1304a2f911cb1b285436411] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=36.0 K 2024-11-11T22:16:41,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T22:16:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,649 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:16:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:41,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:41,651 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ed7e4bae454b1cb4fcd6cb8edd1688, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363399645 2024-11-11T22:16:41,652 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting dcfa020f1dc941c8ae46e7262629a2c7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363399645 2024-11-11T22:16:41,653 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f25270bd51e848d4b9376a4442f9da70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731363399789 2024-11-11T22:16:41,653 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6860f2169a4c07b3cebdc7cca8d614, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731363399789 2024-11-11T22:16:41,654 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b0df686c180242f0b92df8b3c7f72e46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363400442 2024-11-11T22:16:41,654 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b67502a2d1304a2f911cb1b285436411, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363400442 2024-11-11T22:16:41,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/b42e506cd89741388833b9c194ccb639 is 50, key is test_row_0/A:col10/1731363401133/Put/seqid=0 2024-11-11T22:16:41,675 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:41,676 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/62a0a752f080402b89da0f05997fdd08 is 50, key is test_row_0/B:col10/1731363400442/Put/seqid=0 2024-11-11T22:16:41,676 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:41,677 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/af1f37bf88594293b5a1e67524d7cf7a is 50, key is test_row_0/A:col10/1731363400442/Put/seqid=0 2024-11-11T22:16:41,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741882_1058 (size=12663) 2024-11-11T22:16:41,713 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/62a0a752f080402b89da0f05997fdd08 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/62a0a752f080402b89da0f05997fdd08 2024-11-11T22:16:41,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741884_1060 (size=12151) 2024-11-11T22:16:41,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741883_1059 (size=12663) 2024-11-11T22:16:41,725 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into 62a0a752f080402b89da0f05997fdd08(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:41,725 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:41,725 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=13, startTime=1731363401645; duration=0sec 2024-11-11T22:16:41,725 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:41,725 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:41,725 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:41,728 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:41,728 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:41,729 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:41,729 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/38ca90ee19a948ef92688cc96423e441, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/91dd2d5a0939434a8e2804c413aac3f5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9a0b3ec157624085bf92279f8de8e4e8] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=36.0 K 2024-11-11T22:16:41,730 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ca90ee19a948ef92688cc96423e441, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363399645 2024-11-11T22:16:41,732 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 91dd2d5a0939434a8e2804c413aac3f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731363399789 2024-11-11T22:16:41,737 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0b3ec157624085bf92279f8de8e4e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363400442 2024-11-11T22:16:41,738 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/af1f37bf88594293b5a1e67524d7cf7a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/af1f37bf88594293b5a1e67524d7cf7a 2024-11-11T22:16:41,756 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into af1f37bf88594293b5a1e67524d7cf7a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:41,756 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:41,756 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=13, startTime=1731363401645; duration=0sec 2024-11-11T22:16:41,756 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:41,757 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:41,775 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:41,776 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/80187ee7a54740a390838b71a1d60fbd is 50, key is test_row_0/C:col10/1731363400442/Put/seqid=0 2024-11-11T22:16:41,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:16:41,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741885_1061 (size=12663) 2024-11-11T22:16:41,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363461791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363461795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363461795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363461795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363461801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,828 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/80187ee7a54740a390838b71a1d60fbd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/80187ee7a54740a390838b71a1d60fbd 2024-11-11T22:16:41,841 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into 80187ee7a54740a390838b71a1d60fbd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:41,841 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:41,841 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=13, startTime=1731363401645; duration=0sec 2024-11-11T22:16:41,842 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:41,842 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:41,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363461909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363461910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363461910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363461910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:41,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363461919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,120 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/b42e506cd89741388833b9c194ccb639 2024-11-11T22:16:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363462119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363462119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363462136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/fe9e7926b74c4ed592770eb27fa8552a is 50, key is test_row_0/B:col10/1731363401133/Put/seqid=0 2024-11-11T22:16:42,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363462137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363462136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741886_1062 (size=12151) 2024-11-11T22:16:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:16:42,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363462440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363462440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363462450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363462450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363462450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,553 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/fe9e7926b74c4ed592770eb27fa8552a 2024-11-11T22:16:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/71f3385d115b48df8d429e1d1ccce27a is 50, key is test_row_0/C:col10/1731363401133/Put/seqid=0 2024-11-11T22:16:42,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741887_1063 (size=12151) 2024-11-11T22:16:42,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363462949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363462949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,954 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363462969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363462970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363462970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:42,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:42,976 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/71f3385d115b48df8d429e1d1ccce27a 2024-11-11T22:16:42,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/b42e506cd89741388833b9c194ccb639 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b42e506cd89741388833b9c194ccb639 2024-11-11T22:16:42,994 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b42e506cd89741388833b9c194ccb639, entries=150, sequenceid=246, filesize=11.9 K 2024-11-11T22:16:42,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/fe9e7926b74c4ed592770eb27fa8552a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/fe9e7926b74c4ed592770eb27fa8552a 2024-11-11T22:16:43,010 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/fe9e7926b74c4ed592770eb27fa8552a, entries=150, sequenceid=246, filesize=11.9 K 2024-11-11T22:16:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/71f3385d115b48df8d429e1d1ccce27a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/71f3385d115b48df8d429e1d1ccce27a 2024-11-11T22:16:43,039 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/71f3385d115b48df8d429e1d1ccce27a, entries=150, sequenceid=246, filesize=11.9 K 2024-11-11T22:16:43,057 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3f60f99bf48ef670ca729411903c0391 in 1407ms, sequenceid=246, compaction requested=false 2024-11-11T22:16:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T22:16:43,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T22:16:43,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T22:16:43,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8820 sec 2024-11-11T22:16:43,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8940 sec 2024-11-11T22:16:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T22:16:43,319 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:43,320 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T22:16:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:16:43,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:43,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:43,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:16:43,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T22:16:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:43,478 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:16:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:43,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:43,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:43,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:43,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:43,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/c93a8e7123b34996bbe744817152dfb7 is 50, key is test_row_0/A:col10/1731363401795/Put/seqid=0 2024-11-11T22:16:43,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741888_1064 (size=12251) 2024-11-11T22:16:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:16:43,908 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/c93a8e7123b34996bbe744817152dfb7 2024-11-11T22:16:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:16:43,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:43,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7416a81d6f4a4edbb1cde6f9c7ae23a8 is 50, key is test_row_0/B:col10/1731363401795/Put/seqid=0 2024-11-11T22:16:43,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741889_1065 (size=12251) 2024-11-11T22:16:44,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363464009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363464009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363464012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363464013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363464015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363464129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363464130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363464130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363464130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363464130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363464339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363464339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363464339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363464339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363464340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,383 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7416a81d6f4a4edbb1cde6f9c7ae23a8 2024-11-11T22:16:44,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/4d9d5a2aaaf947548395c96db5938e6b is 50, key is test_row_0/C:col10/1731363401795/Put/seqid=0 2024-11-11T22:16:44,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741890_1066 (size=12251) 2024-11-11T22:16:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:16:44,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363464649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363464650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363464650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363464650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363464650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:44,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:44,799 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/4d9d5a2aaaf947548395c96db5938e6b 2024-11-11T22:16:44,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/c93a8e7123b34996bbe744817152dfb7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c93a8e7123b34996bbe744817152dfb7 2024-11-11T22:16:44,834 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c93a8e7123b34996bbe744817152dfb7, entries=150, sequenceid=261, filesize=12.0 K 2024-11-11T22:16:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7416a81d6f4a4edbb1cde6f9c7ae23a8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7416a81d6f4a4edbb1cde6f9c7ae23a8 2024-11-11T22:16:44,848 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7416a81d6f4a4edbb1cde6f9c7ae23a8, entries=150, sequenceid=261, filesize=12.0 K 2024-11-11T22:16:44,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/4d9d5a2aaaf947548395c96db5938e6b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/4d9d5a2aaaf947548395c96db5938e6b 2024-11-11T22:16:44,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/4d9d5a2aaaf947548395c96db5938e6b, entries=150, sequenceid=261, filesize=12.0 K 2024-11-11T22:16:44,861 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3f60f99bf48ef670ca729411903c0391 in 1383ms, sequenceid=261, compaction requested=true 2024-11-11T22:16:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T22:16:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T22:16:44,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T22:16:44,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-11-11T22:16:44,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5500 sec 2024-11-11T22:16:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:45,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363403998, max timestamp=1731363405166Num uniques 859; 2024-11-11T22:16:45,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:45,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=102.87 KB heapSize=301.95 KB 2024-11-11T22:16:45,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:45,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:45,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:45,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:45,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:45,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:45,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3170634999dc4e3f959945845bce4673 is 50, key is test_row_0/A:col10/1731363404012/Put/seqid=0 2024-11-11T22:16:45,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363465194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741891_1067 (size=12301) 2024-11-11T22:16:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3170634999dc4e3f959945845bce4673 2024-11-11T22:16:45,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363465198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363465200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363465202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363465209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/1f7e4eba2f804a63b54619d973875960 is 50, key is test_row_0/B:col10/1731363404012/Put/seqid=0 2024-11-11T22:16:45,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741892_1068 (size=12301) 2024-11-11T22:16:45,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363465319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363465330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363465330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363465330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363465330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T22:16:45,470 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:45,470 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:45,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T22:16:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:16:45,477 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:45,479 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:45,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:45,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363465529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363465539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363465540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363465540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363465549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:16:45,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T22:16:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:45,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:45,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:45,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:45,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/1f7e4eba2f804a63b54619d973875960 2024-11-11T22:16:45,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/556e7620999341e488d5536cf09ad74f is 50, key is test_row_0/C:col10/1731363404012/Put/seqid=0 2024-11-11T22:16:45,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741893_1069 (size=12301) 2024-11-11T22:16:45,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/556e7620999341e488d5536cf09ad74f 2024-11-11T22:16:45,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/3170634999dc4e3f959945845bce4673 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3170634999dc4e3f959945845bce4673 2024-11-11T22:16:45,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3170634999dc4e3f959945845bce4673, entries=150, sequenceid=284, filesize=12.0 K 2024-11-11T22:16:45,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/1f7e4eba2f804a63b54619d973875960 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/1f7e4eba2f804a63b54619d973875960 2024-11-11T22:16:45,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/1f7e4eba2f804a63b54619d973875960, entries=150, sequenceid=284, filesize=12.0 K 2024-11-11T22:16:45,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/556e7620999341e488d5536cf09ad74f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/556e7620999341e488d5536cf09ad74f 2024-11-11T22:16:45,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/556e7620999341e488d5536cf09ad74f, entries=150, sequenceid=284, filesize=12.0 K 2024-11-11T22:16:45,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~301.91 KB/309152, currentSize=73.80 KB/75570 for 3f60f99bf48ef670ca729411903c0391 in 592ms, sequenceid=284, compaction requested=true 2024-11-11T22:16:45,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:45,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:45,763 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:16:45,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:45,763 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:16:45,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:45,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:45,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:45,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:45,765 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:16:45,765 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:45,765 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:45,765 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:16:45,765 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:45,765 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/62a0a752f080402b89da0f05997fdd08, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/fe9e7926b74c4ed592770eb27fa8552a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7416a81d6f4a4edbb1cde6f9c7ae23a8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/1f7e4eba2f804a63b54619d973875960] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=48.2 K 2024-11-11T22:16:45,765 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:45,765 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/af1f37bf88594293b5a1e67524d7cf7a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b42e506cd89741388833b9c194ccb639, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c93a8e7123b34996bbe744817152dfb7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3170634999dc4e3f959945845bce4673] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=48.2 K 2024-11-11T22:16:45,765 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 62a0a752f080402b89da0f05997fdd08, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363400442 2024-11-11T22:16:45,766 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting af1f37bf88594293b5a1e67524d7cf7a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363400442 2024-11-11T22:16:45,766 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9e7926b74c4ed592770eb27fa8552a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731363401133 2024-11-11T22:16:45,766 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b42e506cd89741388833b9c194ccb639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731363401133 2024-11-11T22:16:45,766 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7416a81d6f4a4edbb1cde6f9c7ae23a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731363401775 2024-11-11T22:16:45,767 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c93a8e7123b34996bbe744817152dfb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731363401775 2024-11-11T22:16:45,767 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f7e4eba2f804a63b54619d973875960, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731363404012 2024-11-11T22:16:45,768 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3170634999dc4e3f959945845bce4673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731363404012 2024-11-11T22:16:45,787 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:45,787 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/a287774b861b4900b185e377bfeaddca is 50, key is test_row_0/A:col10/1731363404012/Put/seqid=0 2024-11-11T22:16:45,789 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:45,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T22:16:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:16:45,790 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T22:16:45,790 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/e77f83f1d5ce445d8cdccbbcf9cc171f is 50, key is test_row_0/B:col10/1731363404012/Put/seqid=0 2024-11-11T22:16:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:45,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4887b95cd8ae42148b7c47a5e076b219 is 50, key is test_row_0/A:col10/1731363405196/Put/seqid=0 2024-11-11T22:16:45,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741894_1070 (size=12949) 2024-11-11T22:16:45,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741895_1071 (size=12949) 2024-11-11T22:16:45,817 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/a287774b861b4900b185e377bfeaddca as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/a287774b861b4900b185e377bfeaddca 2024-11-11T22:16:45,819 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/e77f83f1d5ce445d8cdccbbcf9cc171f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e77f83f1d5ce445d8cdccbbcf9cc171f 2024-11-11T22:16:45,830 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into e77f83f1d5ce445d8cdccbbcf9cc171f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:45,830 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:45,830 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=12, startTime=1731363405763; duration=0sec 2024-11-11T22:16:45,830 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:45,830 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:45,830 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:16:45,832 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:16:45,832 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:45,832 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:45,833 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/80187ee7a54740a390838b71a1d60fbd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/71f3385d115b48df8d429e1d1ccce27a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/4d9d5a2aaaf947548395c96db5938e6b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/556e7620999341e488d5536cf09ad74f] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=48.2 K 2024-11-11T22:16:45,833 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 80187ee7a54740a390838b71a1d60fbd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363400442 2024-11-11T22:16:45,833 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into a287774b861b4900b185e377bfeaddca(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:45,833 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:45,833 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=12, startTime=1731363405763; duration=0sec 2024-11-11T22:16:45,833 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:45,834 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:45,834 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 71f3385d115b48df8d429e1d1ccce27a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731363401133 2024-11-11T22:16:45,834 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d9d5a2aaaf947548395c96db5938e6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731363401775 2024-11-11T22:16:45,835 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 556e7620999341e488d5536cf09ad74f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731363404012 2024-11-11T22:16:45,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741896_1072 (size=12301) 2024-11-11T22:16:45,839 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4887b95cd8ae42148b7c47a5e076b219 2024-11-11T22:16:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:45,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:45,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/c2eddc77657e4d8e8772e9196c8417c1 is 50, key is test_row_0/B:col10/1731363405196/Put/seqid=0 2024-11-11T22:16:45,871 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:45,872 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/6c54fe0e006546ebad932f83651136fd is 50, key is test_row_0/C:col10/1731363404012/Put/seqid=0 2024-11-11T22:16:45,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363465879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363465882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363465885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363465887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363465888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:45,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741897_1073 (size=12301) 2024-11-11T22:16:45,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741898_1074 (size=12949) 2024-11-11T22:16:45,920 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/6c54fe0e006546ebad932f83651136fd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6c54fe0e006546ebad932f83651136fd 2024-11-11T22:16:45,931 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into 6c54fe0e006546ebad932f83651136fd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:45,931 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:45,931 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=12, startTime=1731363405764; duration=0sec 2024-11-11T22:16:45,931 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:45,931 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363466000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363466000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363466001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363466001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363466001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:16:46,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363466210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363466210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363466211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363466211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363466211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,306 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/c2eddc77657e4d8e8772e9196c8417c1 2024-11-11T22:16:46,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9f6bb3d2751c4e30adfd7278331ac97d is 50, key is test_row_0/C:col10/1731363405196/Put/seqid=0 2024-11-11T22:16:46,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741899_1075 (size=12301) 2024-11-11T22:16:46,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363466520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363466520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363466520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363466520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363466541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:16:46,760 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9f6bb3d2751c4e30adfd7278331ac97d 2024-11-11T22:16:46,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4887b95cd8ae42148b7c47a5e076b219 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4887b95cd8ae42148b7c47a5e076b219 2024-11-11T22:16:46,797 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4887b95cd8ae42148b7c47a5e076b219, entries=150, sequenceid=298, filesize=12.0 K 2024-11-11T22:16:46,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/c2eddc77657e4d8e8772e9196c8417c1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/c2eddc77657e4d8e8772e9196c8417c1 2024-11-11T22:16:46,826 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/c2eddc77657e4d8e8772e9196c8417c1, entries=150, sequenceid=298, filesize=12.0 K 2024-11-11T22:16:46,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9f6bb3d2751c4e30adfd7278331ac97d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9f6bb3d2751c4e30adfd7278331ac97d 2024-11-11T22:16:46,861 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9f6bb3d2751c4e30adfd7278331ac97d, entries=150, sequenceid=298, filesize=12.0 K 2024-11-11T22:16:46,864 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3f60f99bf48ef670ca729411903c0391 in 1074ms, sequenceid=298, compaction requested=false 2024-11-11T22:16:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T22:16:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T22:16:46,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T22:16:46,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-11-11T22:16:46,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4150 sec 2024-11-11T22:16:47,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:47,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:16:47,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:47,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:47,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/ce974c0acc744019aed963e88602d5fe is 50, key is test_row_0/A:col10/1731363407042/Put/seqid=0 2024-11-11T22:16:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363467066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363467075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363467081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363467081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363467083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741900_1076 (size=14741) 2024-11-11T22:16:47,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363467190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363467199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363467211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363467211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363467211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363467401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363467411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363467425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363467425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363467437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/ce974c0acc744019aed963e88602d5fe 2024-11-11T22:16:47,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/e07b626d0cc846c5beee8d37020f6267 is 50, key is test_row_0/B:col10/1731363407042/Put/seqid=0 2024-11-11T22:16:47,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741901_1077 (size=12301) 2024-11-11T22:16:47,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/e07b626d0cc846c5beee8d37020f6267 2024-11-11T22:16:47,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/f4b8785092e94297a7eb0807fd930bd6 is 50, key is test_row_0/C:col10/1731363407042/Put/seqid=0 2024-11-11T22:16:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T22:16:47,633 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:47,636 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T22:16:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:16:47,644 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:47,646 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:47,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741902_1078 (size=12301) 2024-11-11T22:16:47,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/f4b8785092e94297a7eb0807fd930bd6 2024-11-11T22:16:47,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/ce974c0acc744019aed963e88602d5fe as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ce974c0acc744019aed963e88602d5fe 2024-11-11T22:16:47,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ce974c0acc744019aed963e88602d5fe, entries=200, sequenceid=321, filesize=14.4 K 2024-11-11T22:16:47,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363467710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/e07b626d0cc846c5beee8d37020f6267 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e07b626d0cc846c5beee8d37020f6267 2024-11-11T22:16:47,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363467720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e07b626d0cc846c5beee8d37020f6267, entries=150, sequenceid=321, filesize=12.0 K 2024-11-11T22:16:47,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363467741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/f4b8785092e94297a7eb0807fd930bd6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/f4b8785092e94297a7eb0807fd930bd6 2024-11-11T22:16:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:16:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363467750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363467752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:47,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:47,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:47,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/f4b8785092e94297a7eb0807fd930bd6, entries=150, sequenceid=321, filesize=12.0 K 2024-11-11T22:16:47,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3f60f99bf48ef670ca729411903c0391 in 749ms, sequenceid=321, compaction requested=true 2024-11-11T22:16:47,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:47,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:47,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:47,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:47,794 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:47,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:47,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:47,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:16:47,794 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:47,795 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:47,795 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:47,795 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:47,795 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:47,795 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:47,795 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:47,796 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/a287774b861b4900b185e377bfeaddca, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4887b95cd8ae42148b7c47a5e076b219, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ce974c0acc744019aed963e88602d5fe] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=39.1 K 2024-11-11T22:16:47,796 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e77f83f1d5ce445d8cdccbbcf9cc171f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/c2eddc77657e4d8e8772e9196c8417c1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e07b626d0cc846c5beee8d37020f6267] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=36.7 K 2024-11-11T22:16:47,796 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e77f83f1d5ce445d8cdccbbcf9cc171f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731363404012 2024-11-11T22:16:47,796 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a287774b861b4900b185e377bfeaddca, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731363404012 2024-11-11T22:16:47,796 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c2eddc77657e4d8e8772e9196c8417c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731363405196 2024-11-11T22:16:47,797 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4887b95cd8ae42148b7c47a5e076b219, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731363405196 2024-11-11T22:16:47,799 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce974c0acc744019aed963e88602d5fe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731363405881 2024-11-11T22:16:47,800 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e07b626d0cc846c5beee8d37020f6267, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731363405885 2024-11-11T22:16:47,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T22:16:47,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:47,804 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:16:47,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:47,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:47,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/119445c2ac034db781fcc8fecb344901 is 50, key is test_row_0/A:col10/1731363407072/Put/seqid=0 2024-11-11T22:16:47,887 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:47,888 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/9c193a48696e4c94b9b1dfc0de638a03 is 50, key is test_row_0/B:col10/1731363407042/Put/seqid=0 2024-11-11T22:16:47,898 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#68 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:47,899 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/fa00d81fd2364f3eb61ccaab9678bf38 is 50, key is test_row_0/A:col10/1731363407042/Put/seqid=0 2024-11-11T22:16:47,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741903_1079 (size=12301) 2024-11-11T22:16:47,920 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/119445c2ac034db781fcc8fecb344901 2024-11-11T22:16:47,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741905_1081 (size=13051) 2024-11-11T22:16:47,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/eb2001fc3f744e1c89d743b8df641548 is 50, key is test_row_0/B:col10/1731363407072/Put/seqid=0 2024-11-11T22:16:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:16:47,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741904_1080 (size=13051) 2024-11-11T22:16:47,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741906_1082 (size=12301) 2024-11-11T22:16:47,981 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/eb2001fc3f744e1c89d743b8df641548 2024-11-11T22:16:48,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/7e1b15f3e0ff40059d504b7e4d7c5be5 is 50, key is test_row_0/C:col10/1731363407072/Put/seqid=0 2024-11-11T22:16:48,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741907_1083 (size=12301) 2024-11-11T22:16:48,083 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/7e1b15f3e0ff40059d504b7e4d7c5be5 2024-11-11T22:16:48,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/119445c2ac034db781fcc8fecb344901 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/119445c2ac034db781fcc8fecb344901 2024-11-11T22:16:48,157 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/119445c2ac034db781fcc8fecb344901, entries=150, sequenceid=333, filesize=12.0 K 2024-11-11T22:16:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/eb2001fc3f744e1c89d743b8df641548 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/eb2001fc3f744e1c89d743b8df641548 2024-11-11T22:16:48,217 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/eb2001fc3f744e1c89d743b8df641548, entries=150, sequenceid=333, filesize=12.0 K 2024-11-11T22:16:48,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/7e1b15f3e0ff40059d504b7e4d7c5be5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7e1b15f3e0ff40059d504b7e4d7c5be5 2024-11-11T22:16:48,231 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7e1b15f3e0ff40059d504b7e4d7c5be5, entries=150, sequenceid=333, filesize=12.0 K 2024-11-11T22:16:48,252 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for 3f60f99bf48ef670ca729411903c0391 in 448ms, sequenceid=333, compaction requested=true 2024-11-11T22:16:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T22:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T22:16:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:16:48,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T22:16:48,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 613 msec 2024-11-11T22:16:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:48,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:16:48,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:48,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:48,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:48,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:48,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:48,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:48,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 649 msec 2024-11-11T22:16:48,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f5bd41da6d6449068896ec7c7b07c8cb is 50, key is test_row_0/A:col10/1731363408278/Put/seqid=0 2024-11-11T22:16:48,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741908_1084 (size=14737) 2024-11-11T22:16:48,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f5bd41da6d6449068896ec7c7b07c8cb 2024-11-11T22:16:48,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/af29b482bd91409aa9bd5cc4f8124b20 is 50, key is test_row_0/B:col10/1731363408278/Put/seqid=0 2024-11-11T22:16:48,376 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/fa00d81fd2364f3eb61ccaab9678bf38 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fa00d81fd2364f3eb61ccaab9678bf38 2024-11-11T22:16:48,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363408307, max timestamp=1731363408373Num uniques 859; 2024-11-11T22:16:48,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:48,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363408307, max timestamp=1731363408373Num uniques 859; 2024-11-11T22:16:48,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:48,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363408307, max timestamp=1731363408373Num uniques 859; 2024-11-11T22:16:48,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:48,384 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/9c193a48696e4c94b9b1dfc0de638a03 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9c193a48696e4c94b9b1dfc0de638a03 2024-11-11T22:16:48,396 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into 9c193a48696e4c94b9b1dfc0de638a03(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:48,396 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into fa00d81fd2364f3eb61ccaab9678bf38(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:48,396 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:48,396 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:48,396 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=13, startTime=1731363407794; duration=0sec 2024-11-11T22:16:48,396 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=13, startTime=1731363407793; duration=0sec 2024-11-11T22:16:48,397 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:48,397 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:48,397 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:48,397 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:48,397 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:16:48,399 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:16:48,399 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:48,399 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:48,399 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6c54fe0e006546ebad932f83651136fd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9f6bb3d2751c4e30adfd7278331ac97d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/f4b8785092e94297a7eb0807fd930bd6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7e1b15f3e0ff40059d504b7e4d7c5be5] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=48.7 K 2024-11-11T22:16:48,400 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c54fe0e006546ebad932f83651136fd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731363404012 2024-11-11T22:16:48,401 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f6bb3d2751c4e30adfd7278331ac97d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731363405196 2024-11-11T22:16:48,402 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f4b8785092e94297a7eb0807fd930bd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731363405885 2024-11-11T22:16:48,403 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1b15f3e0ff40059d504b7e4d7c5be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731363407072 2024-11-11T22:16:48,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741909_1085 (size=9857) 2024-11-11T22:16:48,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/af29b482bd91409aa9bd5cc4f8124b20 2024-11-11T22:16:48,439 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:48,440 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/67489fff8a054f299da3a16a10f37979 is 50, key is test_row_0/C:col10/1731363407072/Put/seqid=0 2024-11-11T22:16:48,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/17edcb57047a4ed281759dc350e4da01 is 50, key is test_row_0/C:col10/1731363408278/Put/seqid=0 2024-11-11T22:16:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363468431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363468444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363468449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363468454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363468457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741911_1087 (size=9857) 2024-11-11T22:16:48,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/17edcb57047a4ed281759dc350e4da01 2024-11-11T22:16:48,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741910_1086 (size=13085) 2024-11-11T22:16:48,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f5bd41da6d6449068896ec7c7b07c8cb as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f5bd41da6d6449068896ec7c7b07c8cb 2024-11-11T22:16:48,509 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/67489fff8a054f299da3a16a10f37979 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/67489fff8a054f299da3a16a10f37979 2024-11-11T22:16:48,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f5bd41da6d6449068896ec7c7b07c8cb, entries=200, sequenceid=343, filesize=14.4 K 2024-11-11T22:16:48,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/af29b482bd91409aa9bd5cc4f8124b20 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/af29b482bd91409aa9bd5cc4f8124b20 2024-11-11T22:16:48,527 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into 67489fff8a054f299da3a16a10f37979(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:48,527 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:48,527 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=12, startTime=1731363407794; duration=0sec 2024-11-11T22:16:48,534 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:48,534 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:48,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/af29b482bd91409aa9bd5cc4f8124b20, entries=100, sequenceid=343, filesize=9.6 K 2024-11-11T22:16:48,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/17edcb57047a4ed281759dc350e4da01 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/17edcb57047a4ed281759dc350e4da01 2024-11-11T22:16:48,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363468560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/17edcb57047a4ed281759dc350e4da01, entries=100, sequenceid=343, filesize=9.6 K 2024-11-11T22:16:48,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3f60f99bf48ef670ca729411903c0391 in 288ms, sequenceid=343, compaction requested=true 2024-11-11T22:16:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:16:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:48,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T22:16:48,575 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:48,575 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:48,578 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:48,578 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:48,578 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:48,579 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9c193a48696e4c94b9b1dfc0de638a03, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/eb2001fc3f744e1c89d743b8df641548, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/af29b482bd91409aa9bd5cc4f8124b20] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=34.4 K 2024-11-11T22:16:48,580 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c193a48696e4c94b9b1dfc0de638a03, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731363405885 2024-11-11T22:16:48,580 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:48,580 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:48,581 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:48,581 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fa00d81fd2364f3eb61ccaab9678bf38, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/119445c2ac034db781fcc8fecb344901, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f5bd41da6d6449068896ec7c7b07c8cb] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=39.1 K 2024-11-11T22:16:48,581 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting eb2001fc3f744e1c89d743b8df641548, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731363407072 2024-11-11T22:16:48,583 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa00d81fd2364f3eb61ccaab9678bf38, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731363405885 2024-11-11T22:16:48,583 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting af29b482bd91409aa9bd5cc4f8124b20, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731363408278 2024-11-11T22:16:48,584 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 119445c2ac034db781fcc8fecb344901, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731363407072 2024-11-11T22:16:48,585 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5bd41da6d6449068896ec7c7b07c8cb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731363408258 2024-11-11T22:16:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:48,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-11T22:16:48,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:48,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:48,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:48,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:48,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:48,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:48,639 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:48,640 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/915bc9cdba5d4a669cbd3a55f2a4cd6b is 50, key is test_row_0/B:col10/1731363408278/Put/seqid=0 2024-11-11T22:16:48,643 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:48,644 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/743f522259524b3289bb8fd98d59231c is 50, key is test_row_0/A:col10/1731363408278/Put/seqid=0 2024-11-11T22:16:48,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/fef903132bc24ae38ef300e78fca63ac is 50, key is test_row_0/A:col10/1731363408408/Put/seqid=0 2024-11-11T22:16:48,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741913_1089 (size=13153) 2024-11-11T22:16:48,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363468630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,712 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/743f522259524b3289bb8fd98d59231c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/743f522259524b3289bb8fd98d59231c 2024-11-11T22:16:48,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741914_1090 (size=13153) 2024-11-11T22:16:48,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741912_1088 (size=17181) 2024-11-11T22:16:48,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363468699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363468722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363468729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,753 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into 743f522259524b3289bb8fd98d59231c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:48,753 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:48,753 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=13, startTime=1731363408574; duration=0sec 2024-11-11T22:16:48,753 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:48,753 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:48,753 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T22:16:48,763 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T22:16:48,764 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T22:16:48,764 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. because compaction request was cancelled 2024-11-11T22:16:48,764 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T22:16:48,780 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:48,781 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-11T22:16:48,786 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:16:48,787 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:48,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:48,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363468783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363468809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363468849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363468860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363468864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:16:48,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:16:48,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:48,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:48,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:48,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:48,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363469020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363469059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363469079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363469089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:16:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:16:49,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363469109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/fef903132bc24ae38ef300e78fca63ac 2024-11-11T22:16:49,149 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/915bc9cdba5d4a669cbd3a55f2a4cd6b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/915bc9cdba5d4a669cbd3a55f2a4cd6b 2024-11-11T22:16:49,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/9187264dd8fb4bb6a15d819f81d125f0 is 50, key is test_row_0/B:col10/1731363408408/Put/seqid=0 2024-11-11T22:16:49,168 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into 915bc9cdba5d4a669cbd3a55f2a4cd6b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:49,168 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:49,168 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=13, startTime=1731363408574; duration=0sec 2024-11-11T22:16:49,169 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:49,169 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:49,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741915_1091 (size=12301) 2024-11-11T22:16:49,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:16:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:49,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363469344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363469379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363469389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:16:49,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363469409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:16:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:49,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:16:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/9187264dd8fb4bb6a15d819f81d125f0 2024-11-11T22:16:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:49,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/caf08db6f2964452a4996b991bb59daa is 50, key is test_row_0/C:col10/1731363408408/Put/seqid=0 2024-11-11T22:16:49,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363469619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:49,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741916_1092 (size=12301) 2024-11-11T22:16:49,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/caf08db6f2964452a4996b991bb59daa 2024-11-11T22:16:49,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/fef903132bc24ae38ef300e78fca63ac as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fef903132bc24ae38ef300e78fca63ac 2024-11-11T22:16:49,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fef903132bc24ae38ef300e78fca63ac, entries=250, sequenceid=383, filesize=16.8 K 2024-11-11T22:16:49,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/9187264dd8fb4bb6a15d819f81d125f0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9187264dd8fb4bb6a15d819f81d125f0 2024-11-11T22:16:49,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9187264dd8fb4bb6a15d819f81d125f0, entries=150, sequenceid=383, filesize=12.0 K 2024-11-11T22:16:49,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/caf08db6f2964452a4996b991bb59daa as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/caf08db6f2964452a4996b991bb59daa 2024-11-11T22:16:49,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/caf08db6f2964452a4996b991bb59daa, entries=150, sequenceid=383, filesize=12.0 K 2024-11-11T22:16:49,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=40.25 KB/41220 for 3f60f99bf48ef670ca729411903c0391 in 1091ms, sequenceid=383, compaction requested=true 2024-11-11T22:16:49,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:49,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:49,695 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T22:16:49,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:49,696 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T22:16:49,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:49,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:49,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:49,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:49,697 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T22:16:49,697 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T22:16:49,697 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. because compaction request was cancelled 2024-11-11T22:16:49,697 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:49,697 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:49,698 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T22:16:49,698 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T22:16:49,698 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. because compaction request was cancelled 2024-11-11T22:16:49,698 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:49,699 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:49,699 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:49,699 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,699 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/67489fff8a054f299da3a16a10f37979, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/17edcb57047a4ed281759dc350e4da01, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/caf08db6f2964452a4996b991bb59daa] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=34.4 K 2024-11-11T22:16:49,700 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67489fff8a054f299da3a16a10f37979, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731363407072 2024-11-11T22:16:49,700 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17edcb57047a4ed281759dc350e4da01, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731363408278 2024-11-11T22:16:49,701 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf08db6f2964452a4996b991bb59daa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731363408408 2024-11-11T22:16:49,724 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:49,725 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/dbfd73791b9945749423e60817d8c7b9 is 50, key is test_row_0/C:col10/1731363408408/Put/seqid=0 2024-11-11T22:16:49,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T22:16:49,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:49,735 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T22:16:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:49,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741917_1093 (size=13187) 2024-11-11T22:16:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/91af8bd98d074c8381e148d2d7c4870d is 50, key is test_row_0/A:col10/1731363408639/Put/seqid=0 2024-11-11T22:16:49,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741918_1094 (size=12301) 2024-11-11T22:16:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:49,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:16:49,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731363409922, max timestamp=1731363409938Num uniques 859; 2024-11-11T22:16:49,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731363409922, max timestamp=1731363409938Num uniques 859; 2024-11-11T22:16:49,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:16:49,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:16:49,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731363409922, max timestamp=1731363409938Num uniques 859; 2024-11-11T22:16:49,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:16:50,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363469995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363469996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363469997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363469998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363470109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363470109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363470110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363470109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,157 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/91af8bd98d074c8381e148d2d7c4870d 2024-11-11T22:16:50,169 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/dbfd73791b9945749423e60817d8c7b9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/dbfd73791b9945749423e60817d8c7b9 2024-11-11T22:16:50,178 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into dbfd73791b9945749423e60817d8c7b9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:50,178 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:50,178 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=13, startTime=1731363409696; duration=0sec 2024-11-11T22:16:50,178 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:50,178 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:50,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/b9d0370b5a6140bdb5da5b94d7fa7ddd is 50, key is test_row_0/B:col10/1731363408639/Put/seqid=0 2024-11-11T22:16:50,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741919_1095 (size=12301) 2024-11-11T22:16:50,238 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/b9d0370b5a6140bdb5da5b94d7fa7ddd 2024-11-11T22:16:50,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9b75d36da50048a39d03ec2747a10ec8 is 50, key is test_row_0/C:col10/1731363408639/Put/seqid=0 2024-11-11T22:16:50,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741920_1096 (size=12301) 2024-11-11T22:16:50,295 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9b75d36da50048a39d03ec2747a10ec8 2024-11-11T22:16:50,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/91af8bd98d074c8381e148d2d7c4870d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/91af8bd98d074c8381e148d2d7c4870d 2024-11-11T22:16:50,313 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/91af8bd98d074c8381e148d2d7c4870d, entries=150, sequenceid=394, filesize=12.0 K 2024-11-11T22:16:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/b9d0370b5a6140bdb5da5b94d7fa7ddd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b9d0370b5a6140bdb5da5b94d7fa7ddd 2024-11-11T22:16:50,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363470319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363470320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,328 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b9d0370b5a6140bdb5da5b94d7fa7ddd, entries=150, sequenceid=394, filesize=12.0 K 2024-11-11T22:16:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/9b75d36da50048a39d03ec2747a10ec8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9b75d36da50048a39d03ec2747a10ec8 2024-11-11T22:16:50,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363470320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363470320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,340 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9b75d36da50048a39d03ec2747a10ec8, entries=150, sequenceid=394, filesize=12.0 K 2024-11-11T22:16:50,341 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 3f60f99bf48ef670ca729411903c0391 in 606ms, sequenceid=394, compaction requested=true 2024-11-11T22:16:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:50,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-11T22:16:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-11T22:16:50,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T22:16:50,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-11-11T22:16:50,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.5690 sec 2024-11-11T22:16:50,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731363409978, max timestamp=1731363410642Num uniques 1068; 2024-11-11T22:16:50,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T22:16:50,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731363409978, max timestamp=1731363410644Num uniques 1068; 2024-11-11T22:16:50,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T22:16:50,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=67.09 KB heapSize=183.88 KB 2024-11-11T22:16:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:50,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1731363409978, max timestamp=1731363410644Num uniques 1068; 2024-11-11T22:16:50,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48900, new segment data size=7680, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T22:16:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:50,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:50,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4c86394dbf794b45aacb8b5fcb36b13b is 50, key is test_row_0/A:col10/1731363409978/Put/seqid=0 2024-11-11T22:16:50,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741921_1097 (size=12301) 2024-11-11T22:16:50,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363410718, max timestamp=1731363410731Num uniques 859; 2024-11-11T22:16:50,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:50,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363410718, max timestamp=1731363410731Num uniques 859; 2024-11-11T22:16:50,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363410718, max timestamp=1731363410731Num uniques 859; 2024-11-11T22:16:50,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:50,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:50,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731363410754, max timestamp=1731363410794Num uniques 1118; 2024-11-11T22:16:50,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731363410754, max timestamp=1731363410794Num uniques 1118; 2024-11-11T22:16:50,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-11T22:16:50,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-11T22:16:50,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731363410754, max timestamp=1731363410794Num uniques 1118; 2024-11-11T22:16:50,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-11T22:16:50,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731363410826, max timestamp=1731363410848Num uniques 1127; 2024-11-11T22:16:50,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731363410826, max timestamp=1731363410848Num uniques 1127; 2024-11-11T22:16:50,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-11T22:16:50,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-11T22:16:50,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731363410826, max timestamp=1731363410848Num uniques 1027; 2024-11-11T22:16:50,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T22:16:50,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363470899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363470905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=486, cellSize=22256, totalHeapSize=50784, min timestamp=1731363410875, max timestamp=1731363410907Num uniques 1086; 2024-11-11T22:16:50,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=22256, suffix heap size=140912,new segment heap size=50784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=486 2024-11-11T22:16:50,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=486, cellSize=22256, totalHeapSize=50784, min timestamp=1731363410875, max timestamp=1731363410907Num uniques 1086; 2024-11-11T22:16:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=22256, suffix heap size=140912,new segment heap size=50784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=486 2024-11-11T22:16:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363470907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:50,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=486, cellSize=22256, totalHeapSize=50784, min timestamp=1731363410875, max timestamp=1731363410907Num uniques 1086; 2024-11-11T22:16:50,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=22256, suffix heap size=140912,new segment heap size=50784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=486 2024-11-11T22:16:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:50,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T22:16:50,939 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:50,940 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:16:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-11T22:16:50,942 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:16:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:16:50,943 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:16:50,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:16:51,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363470993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363471009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363471020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363471020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363471020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:16:51,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:51,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:51,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4c86394dbf794b45aacb8b5fcb36b13b 2024-11-11T22:16:51,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363471120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363471130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/2cab8f8420cc45039e39a4e65b1bdb9a is 50, key is test_row_0/B:col10/1731363409978/Put/seqid=0 2024-11-11T22:16:51,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741922_1098 (size=12301) 2024-11-11T22:16:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363471229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363471229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363471229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:51,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:16:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363471349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363471360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363471539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363471539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363471540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/2cab8f8420cc45039e39a4e65b1bdb9a 2024-11-11T22:16:51,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:51,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:16:51,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/e33a4091d6fa4150900fea4f0748f1e9 is 50, key is test_row_0/C:col10/1731363409978/Put/seqid=0 2024-11-11T22:16:51,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741923_1099 (size=12301) 2024-11-11T22:16:51,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/e33a4091d6fa4150900fea4f0748f1e9 2024-11-11T22:16:51,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4c86394dbf794b45aacb8b5fcb36b13b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4c86394dbf794b45aacb8b5fcb36b13b 2024-11-11T22:16:51,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4c86394dbf794b45aacb8b5fcb36b13b, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T22:16:51,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/2cab8f8420cc45039e39a4e65b1bdb9a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2cab8f8420cc45039e39a4e65b1bdb9a 2024-11-11T22:16:51,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363471659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2cab8f8420cc45039e39a4e65b1bdb9a, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T22:16:51,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/e33a4091d6fa4150900fea4f0748f1e9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e33a4091d6fa4150900fea4f0748f1e9 2024-11-11T22:16:51,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e33a4091d6fa4150900fea4f0748f1e9, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T22:16:51,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=161.02 KB/164880 for 3f60f99bf48ef670ca729411903c0391 in 1033ms, sequenceid=434, compaction requested=true 2024-11-11T22:16:51,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:51,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:51,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:51,682 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:16:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:51,683 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:16:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:16:51,684 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:16:51,685 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:51,685 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,685 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/915bc9cdba5d4a669cbd3a55f2a4cd6b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9187264dd8fb4bb6a15d819f81d125f0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b9d0370b5a6140bdb5da5b94d7fa7ddd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2cab8f8420cc45039e39a4e65b1bdb9a] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=48.9 K 2024-11-11T22:16:51,685 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:16:51,685 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:51,685 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,686 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915bc9cdba5d4a669cbd3a55f2a4cd6b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731363407075 2024-11-11T22:16:51,686 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/743f522259524b3289bb8fd98d59231c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fef903132bc24ae38ef300e78fca63ac, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/91af8bd98d074c8381e148d2d7c4870d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4c86394dbf794b45aacb8b5fcb36b13b] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=53.6 K 2024-11-11T22:16:51,686 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 743f522259524b3289bb8fd98d59231c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731363407075 2024-11-11T22:16:51,686 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9187264dd8fb4bb6a15d819f81d125f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731363408408 2024-11-11T22:16:51,687 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fef903132bc24ae38ef300e78fca63ac, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731363408408 2024-11-11T22:16:51,687 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9d0370b5a6140bdb5da5b94d7fa7ddd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731363408610 2024-11-11T22:16:51,687 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 91af8bd98d074c8381e148d2d7c4870d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731363408610 2024-11-11T22:16:51,687 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cab8f8420cc45039e39a4e65b1bdb9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731363409978 2024-11-11T22:16:51,688 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c86394dbf794b45aacb8b5fcb36b13b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731363409978 2024-11-11T22:16:51,702 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:51,703 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/c762a6929c5d458197055498c521f655 is 50, key is test_row_0/A:col10/1731363409978/Put/seqid=0 2024-11-11T22:16:51,711 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:51,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=167.72 KB heapSize=453.94 KB 2024-11-11T22:16:51,712 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/3d3f0f797bbd42e69fdff111a164de28 is 50, key is test_row_0/B:col10/1731363409978/Put/seqid=0 2024-11-11T22:16:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:51,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:51,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:51,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/0bd7a141d1e642a7ae5c130845c8e75d is 50, key is test_row_0/A:col10/1731363410944/Put/seqid=0 2024-11-11T22:16:51,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741924_1100 (size=13289) 2024-11-11T22:16:51,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741925_1101 (size=13289) 2024-11-11T22:16:51,773 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/3d3f0f797bbd42e69fdff111a164de28 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d3f0f797bbd42e69fdff111a164de28 2024-11-11T22:16:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363471772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:51,789 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into 3d3f0f797bbd42e69fdff111a164de28(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:51,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:51,790 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=12, startTime=1731363411682; duration=0sec 2024-11-11T22:16:51,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:51,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:51,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:51,792 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:51,792 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:51,792 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,793 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/dbfd73791b9945749423e60817d8c7b9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9b75d36da50048a39d03ec2747a10ec8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e33a4091d6fa4150900fea4f0748f1e9] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=36.9 K 2024-11-11T22:16:51,794 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbfd73791b9945749423e60817d8c7b9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731363408408 2024-11-11T22:16:51,795 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b75d36da50048a39d03ec2747a10ec8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731363408610 2024-11-11T22:16:51,795 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33a4091d6fa4150900fea4f0748f1e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731363409978 2024-11-11T22:16:51,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741926_1102 (size=14741) 2024-11-11T22:16:51,815 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:51,816 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/18938ed185af44dda444e44bfa2d6794 is 50, key is test_row_0/C:col10/1731363409978/Put/seqid=0 2024-11-11T22:16:51,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741927_1103 (size=13289) 2024-11-11T22:16:51,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:51,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:51,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:51,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:51,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363471889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:52,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363472061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363472061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363472063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:52,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:16:52,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363472109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:52,170 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/c762a6929c5d458197055498c521f655 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c762a6929c5d458197055498c521f655 2024-11-11T22:16:52,181 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into c762a6929c5d458197055498c521f655(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:52,181 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:52,181 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=12, startTime=1731363411682; duration=0sec 2024-11-11T22:16:52,182 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:52,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:52,182 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363472169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:52,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:52,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/0bd7a141d1e642a7ae5c130845c8e75d 2024-11-11T22:16:52,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/5aca413cf6914b6796af058bd2b67d42 is 50, key is test_row_0/B:col10/1731363410944/Put/seqid=0 2024-11-11T22:16:52,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741928_1104 (size=12301) 2024-11-11T22:16:52,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/5aca413cf6914b6796af058bd2b67d42 2024-11-11T22:16:52,267 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/18938ed185af44dda444e44bfa2d6794 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/18938ed185af44dda444e44bfa2d6794 2024-11-11T22:16:52,281 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into 18938ed185af44dda444e44bfa2d6794(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:52,281 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:52,281 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=13, startTime=1731363411683; duration=0sec 2024-11-11T22:16:52,281 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:52,281 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:52,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/fd94ac1fb80843ae9823162a15eaa4bd is 50, key is test_row_0/C:col10/1731363410944/Put/seqid=0 2024-11-11T22:16:52,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741929_1105 (size=12301) 2024-11-11T22:16:52,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/fd94ac1fb80843ae9823162a15eaa4bd 2024-11-11T22:16:52,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/0bd7a141d1e642a7ae5c130845c8e75d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/0bd7a141d1e642a7ae5c130845c8e75d 2024-11-11T22:16:52,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/0bd7a141d1e642a7ae5c130845c8e75d, entries=200, sequenceid=521, filesize=14.4 K 2024-11-11T22:16:52,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/5aca413cf6914b6796af058bd2b67d42 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/5aca413cf6914b6796af058bd2b67d42 2024-11-11T22:16:52,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/5aca413cf6914b6796af058bd2b67d42, entries=150, sequenceid=521, filesize=12.0 K 2024-11-11T22:16:52,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/fd94ac1fb80843ae9823162a15eaa4bd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/fd94ac1fb80843ae9823162a15eaa4bd 2024-11-11T22:16:52,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:52,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:52,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/fd94ac1fb80843ae9823162a15eaa4bd, entries=150, sequenceid=521, filesize=12.0 K 2024-11-11T22:16:52,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~453.89 KB/464784, currentSize=20.13 KB/20610 for 3f60f99bf48ef670ca729411903c0391 in 656ms, sequenceid=521, compaction requested=false 2024-11-11T22:16:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:52,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:16:52,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:52,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:52,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:52,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:52,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:52,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:52,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/9b93fa8140044867b5cf26add6707b66 is 50, key is test_row_0/A:col10/1731363412434/Put/seqid=0 2024-11-11T22:16:52,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741930_1106 (size=14741) 2024-11-11T22:16:52,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/9b93fa8140044867b5cf26add6707b66 2024-11-11T22:16:52,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/348dfa30586b401db8ce05d1e5d240e7 is 50, key is test_row_0/B:col10/1731363412434/Put/seqid=0 2024-11-11T22:16:52,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741931_1107 (size=12301) 2024-11-11T22:16:52,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363412605, max timestamp=1731363412670Num uniques 859; 2024-11-11T22:16:52,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363412605, max timestamp=1731363412670Num uniques 859; 2024-11-11T22:16:52,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:52,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363412605, max timestamp=1731363412670Num uniques 859; 2024-11-11T22:16:52,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:52,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:16:52,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363472773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:52,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:52,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363472889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:52,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:52,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:52,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:52,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:52,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/348dfa30586b401db8ce05d1e5d240e7 2024-11-11T22:16:53,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/e8f167efc7ca4f01ba6a10e3e60f926a is 50, key is test_row_0/C:col10/1731363412434/Put/seqid=0 2024-11-11T22:16:53,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48508 deadline: 1731363473080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:16:53,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48524 deadline: 1731363473080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48492 deadline: 1731363473080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,094 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 3f60f99bf48ef670ca729411903c0391 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:16:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,096 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 3f60f99bf48ef670ca729411903c0391 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:16:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,096 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_1' in 3f60f99bf48ef670ca729411903c0391 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:16:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363473100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741932_1108 (size=12301) 2024-11-11T22:16:53,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/e8f167efc7ca4f01ba6a10e3e60f926a 2024-11-11T22:16:53,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/9b93fa8140044867b5cf26add6707b66 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/9b93fa8140044867b5cf26add6707b66 2024-11-11T22:16:53,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/9b93fa8140044867b5cf26add6707b66, entries=200, sequenceid=534, filesize=14.4 K 2024-11-11T22:16:53,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:53,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:53,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/348dfa30586b401db8ce05d1e5d240e7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/348dfa30586b401db8ce05d1e5d240e7 2024-11-11T22:16:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/348dfa30586b401db8ce05d1e5d240e7, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T22:16:53,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/e8f167efc7ca4f01ba6a10e3e60f926a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e8f167efc7ca4f01ba6a10e3e60f926a 2024-11-11T22:16:53,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e8f167efc7ca4f01ba6a10e3e60f926a, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T22:16:53,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3f60f99bf48ef670ca729411903c0391 in 718ms, sequenceid=534, compaction requested=true 2024-11-11T22:16:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:16:53,170 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:53,170 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:16:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f60f99bf48ef670ca729411903c0391:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:16:53,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:53,172 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:53,172 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/A is initiating minor compaction (all files) 2024-11-11T22:16:53,172 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/A in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,173 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c762a6929c5d458197055498c521f655, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/0bd7a141d1e642a7ae5c130845c8e75d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/9b93fa8140044867b5cf26add6707b66] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=41.8 K 2024-11-11T22:16:53,173 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:53,173 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/B is initiating minor compaction (all files) 2024-11-11T22:16:53,173 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/B in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,173 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d3f0f797bbd42e69fdff111a164de28, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/5aca413cf6914b6796af058bd2b67d42, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/348dfa30586b401db8ce05d1e5d240e7] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=37.0 K 2024-11-11T22:16:53,174 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c762a6929c5d458197055498c521f655, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731363409978 2024-11-11T22:16:53,174 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3f0f797bbd42e69fdff111a164de28, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731363409978 2024-11-11T22:16:53,174 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd7a141d1e642a7ae5c130845c8e75d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731363410944 2024-11-11T22:16:53,175 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aca413cf6914b6796af058bd2b67d42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731363410944 2024-11-11T22:16:53,175 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b93fa8140044867b5cf26add6707b66, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731363412427 2024-11-11T22:16:53,175 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 348dfa30586b401db8ce05d1e5d240e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731363412434 2024-11-11T22:16:53,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731363412738, max timestamp=1731363413190Num uniques 1018; 2024-11-11T22:16:53,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731363412738, max timestamp=1731363413190Num uniques 1018; 2024-11-11T22:16:53,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T22:16:53,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731363412738, max timestamp=1731363413190Num uniques 1018; 2024-11-11T22:16:53,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T22:16:53,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T22:16:53,207 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#A#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:53,208 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/5e03e75d551b40a988e4f5e46fd32394 is 50, key is test_row_0/A:col10/1731363412434/Put/seqid=0 2024-11-11T22:16:53,214 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:53,215 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/71fdea5667ca4e68a17ebe9f40b8b087 is 50, key is test_row_0/B:col10/1731363412434/Put/seqid=0 2024-11-11T22:16:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:53,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-11T22:16:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:53,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:53,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f899e63ddc474afaae6336e6298a847b is 50, key is test_row_0/A:col10/1731363413209/Put/seqid=0 2024-11-11T22:16:53,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:53,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:53,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741933_1109 (size=13391) 2024-11-11T22:16:53,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741934_1110 (size=13391) 2024-11-11T22:16:53,354 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/71fdea5667ca4e68a17ebe9f40b8b087 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/71fdea5667ca4e68a17ebe9f40b8b087 2024-11-11T22:16:53,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741935_1111 (size=14741) 2024-11-11T22:16:53,367 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/B of 3f60f99bf48ef670ca729411903c0391 into 71fdea5667ca4e68a17ebe9f40b8b087(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:53,367 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:53,367 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/B, priority=13, startTime=1731363413170; duration=0sec 2024-11-11T22:16:53,367 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:16:53,367 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:B 2024-11-11T22:16:53,367 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:16:53,369 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:16:53,369 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 3f60f99bf48ef670ca729411903c0391/C is initiating minor compaction (all files) 2024-11-11T22:16:53,369 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f60f99bf48ef670ca729411903c0391/C in TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,369 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/18938ed185af44dda444e44bfa2d6794, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/fd94ac1fb80843ae9823162a15eaa4bd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e8f167efc7ca4f01ba6a10e3e60f926a] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp, totalSize=37.0 K 2024-11-11T22:16:53,370 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 18938ed185af44dda444e44bfa2d6794, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731363409978 2024-11-11T22:16:53,370 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fd94ac1fb80843ae9823162a15eaa4bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731363410944 2024-11-11T22:16:53,371 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f167efc7ca4f01ba6a10e3e60f926a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731363412434 2024-11-11T22:16:53,394 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f60f99bf48ef670ca729411903c0391#C#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:16:53,395 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/5c5c1e2e5b914ca8859c6727713a278a is 50, key is test_row_0/C:col10/1731363412434/Put/seqid=0 2024-11-11T22:16:53,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741936_1112 (size=13391) 2024-11-11T22:16:53,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363413327, max timestamp=1731363413422Num uniques 859; 2024-11-11T22:16:53,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:53,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363413327, max timestamp=1731363413422Num uniques 859; 2024-11-11T22:16:53,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363413327, max timestamp=1731363413422Num uniques 859; 2024-11-11T22:16:53,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:53,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:16:53,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:53,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:53,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363473549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363473551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:53,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:53,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363473669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363473669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,733 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/5e03e75d551b40a988e4f5e46fd32394 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e03e75d551b40a988e4f5e46fd32394 2024-11-11T22:16:53,745 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/A of 3f60f99bf48ef670ca729411903c0391 into 5e03e75d551b40a988e4f5e46fd32394(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:53,745 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:53,745 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/A, priority=13, startTime=1731363413170; duration=0sec 2024-11-11T22:16:53,747 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:53,747 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:A 2024-11-11T22:16:53,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f899e63ddc474afaae6336e6298a847b 2024-11-11T22:16:53,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7b05a9c7a38342aab6faeef328794cc3 is 50, key is test_row_0/B:col10/1731363413209/Put/seqid=0 2024-11-11T22:16:53,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741937_1113 (size=12301) 2024-11-11T22:16:53,825 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/5c5c1e2e5b914ca8859c6727713a278a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/5c5c1e2e5b914ca8859c6727713a278a 2024-11-11T22:16:53,832 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f60f99bf48ef670ca729411903c0391/C of 3f60f99bf48ef670ca729411903c0391 into 5c5c1e2e5b914ca8859c6727713a278a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:16:53,832 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:53,832 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., storeName=3f60f99bf48ef670ca729411903c0391/C, priority=13, startTime=1731363413170; duration=0sec 2024-11-11T22:16:53,833 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:16:53,833 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f60f99bf48ef670ca729411903c0391:C 2024-11-11T22:16:53,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363473879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363473880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:53,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:53,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:54,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:54,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7b05a9c7a38342aab6faeef328794cc3 2024-11-11T22:16:54,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363474191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:54,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363474191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:54,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/be6e2f0608c8469ba1ec7a909a29f569 is 50, key is test_row_0/C:col10/1731363413209/Put/seqid=0 2024-11-11T22:16:54,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741938_1114 (size=12301) 2024-11-11T22:16:54,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:54,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:54,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:54,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/be6e2f0608c8469ba1ec7a909a29f569 2024-11-11T22:16:54,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/f899e63ddc474afaae6336e6298a847b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f899e63ddc474afaae6336e6298a847b 2024-11-11T22:16:54,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:54,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:16:54,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f899e63ddc474afaae6336e6298a847b, entries=200, sequenceid=576, filesize=14.4 K 2024-11-11T22:16:54,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48550 deadline: 1731363474700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:54,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:16:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48504 deadline: 1731363474700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:16:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:16:54,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7b05a9c7a38342aab6faeef328794cc3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7b05a9c7a38342aab6faeef328794cc3 2024-11-11T22:16:54,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7b05a9c7a38342aab6faeef328794cc3, entries=150, sequenceid=576, filesize=12.0 K 2024-11-11T22:16:54,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/be6e2f0608c8469ba1ec7a909a29f569 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/be6e2f0608c8469ba1ec7a909a29f569 2024-11-11T22:16:54,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/be6e2f0608c8469ba1ec7a909a29f569, entries=150, sequenceid=576, filesize=12.0 K 2024-11-11T22:16:54,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=140.89 KB/144270 for 3f60f99bf48ef670ca729411903c0391 in 1526ms, sequenceid=576, compaction requested=false 2024-11-11T22:16:54,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:54,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-11T22:16:54,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:54,848 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-11T22:16:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:16:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/e1bf747380ec4364a4e1c5f58bbe6a5b is 50, key is test_row_0/A:col10/1731363413527/Put/seqid=0 2024-11-11T22:16:54,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741939_1115 (size=12301) 2024-11-11T22:16:54,880 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/e1bf747380ec4364a4e1c5f58bbe6a5b 2024-11-11T22:16:54,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7e86760b562f4a4ba76533cd2a8651d7 is 50, key is test_row_0/B:col10/1731363413527/Put/seqid=0 2024-11-11T22:16:54,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741940_1116 (size=12301) 2024-11-11T22:16:54,926 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7e86760b562f4a4ba76533cd2a8651d7 2024-11-11T22:16:54,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/353012ef37f444df86558c08d30b6f41 is 50, key is test_row_0/C:col10/1731363413527/Put/seqid=0 2024-11-11T22:16:54,977 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T22:16:54,977 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:54,979 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T22:16:54,979 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:54,984 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:54,984 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:54,984 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T22:16:54,984 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:54,984 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:54,984 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:54,984 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:54,984 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:54,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:54,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:54,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:54,992 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T22:16:54,992 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:54,992 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:54,992 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:54,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741941_1117 (size=12301) 2024-11-11T22:16:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:16:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:55,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. as already flushing 2024-11-11T22:16:55,181 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T22:16:55,181 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:55,181 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:55,181 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,182 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T22:16:55,182 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:55,182 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T22:16:55,182 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,182 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:55,182 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,182 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:55,182 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:55,395 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/353012ef37f444df86558c08d30b6f41 2024-11-11T22:16:55,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/e1bf747380ec4364a4e1c5f58bbe6a5b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/e1bf747380ec4364a4e1c5f58bbe6a5b 2024-11-11T22:16:55,405 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/e1bf747380ec4364a4e1c5f58bbe6a5b, entries=150, sequenceid=615, filesize=12.0 K 2024-11-11T22:16:55,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7e86760b562f4a4ba76533cd2a8651d7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7e86760b562f4a4ba76533cd2a8651d7 2024-11-11T22:16:55,412 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7e86760b562f4a4ba76533cd2a8651d7, entries=150, sequenceid=615, filesize=12.0 K 2024-11-11T22:16:55,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/353012ef37f444df86558c08d30b6f41 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/353012ef37f444df86558c08d30b6f41 2024-11-11T22:16:55,419 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/353012ef37f444df86558c08d30b6f41, entries=150, sequenceid=615, filesize=12.0 K 2024-11-11T22:16:55,420 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=20.13 KB/20610 for 3f60f99bf48ef670ca729411903c0391 in 572ms, sequenceid=615, compaction requested=true 2024-11-11T22:16:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 3f60f99bf48ef670ca729411903c0391: 2024-11-11T22:16:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-11T22:16:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-11T22:16:55,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T22:16:55,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4790 sec 2024-11-11T22:16:55,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 4.4860 sec 2024-11-11T22:16:55,721 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T22:16:55,721 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T22:16:55,721 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:55,721 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:16:55,721 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,721 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,721 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,721 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:55,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:55,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:56,250 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:16:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T22:16:59,140 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2014 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1960 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:16:59,140 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 931 2024-11-11T22:16:59,141 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2792 rows 2024-11-11T22:16:59,141 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 933 2024-11-11T22:16:59,141 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2799 rows 2024-11-11T22:16:59,141 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:16:59,141 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:16:59,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:59,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:16:59,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:16:59,145 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:16:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:16:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:16:59,153 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363419152"}]},"ts":"1731363419152"} 2024-11-11T22:16:59,155 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:16:59,155 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:16:59,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:16:59,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, UNASSIGN}] 2024-11-11T22:16:59,162 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, UNASSIGN 2024-11-11T22:16:59,163 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3f60f99bf48ef670ca729411903c0391, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:16:59,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, UNASSIGN because future has completed 2024-11-11T22:16:59,167 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:16:59,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:16:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:16:59,324 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:16:59,324 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:16:59,325 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 3f60f99bf48ef670ca729411903c0391, disabling compactions & flushes 2024-11-11T22:16:59,325 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:59,325 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:59,325 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. after waiting 0 ms 2024-11-11T22:16:59,325 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:16:59,325 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 3f60f99bf48ef670ca729411903c0391 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T22:16:59,325 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=A 2024-11-11T22:16:59,326 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:59,326 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=B 2024-11-11T22:16:59,326 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:59,326 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3f60f99bf48ef670ca729411903c0391, store=C 2024-11-11T22:16:59,326 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:16:59,331 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4fbd4545c9984b0aad25585920ec3572 is 50, key is test_row_0/A:col10/1731363415720/Put/seqid=0 2024-11-11T22:16:59,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741942_1118 (size=12301) 2024-11-11T22:16:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:16:59,746 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4fbd4545c9984b0aad25585920ec3572 2024-11-11T22:16:59,774 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7b3373198e0941c9bcb837aa87de46f5 is 50, key is test_row_0/B:col10/1731363415720/Put/seqid=0 2024-11-11T22:16:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:16:59,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741943_1119 (size=12301) 2024-11-11T22:17:00,188 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7b3373198e0941c9bcb837aa87de46f5 2024-11-11T22:17:00,201 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/a47ef9745d52496f85d3777d0b83919e is 50, key is test_row_0/C:col10/1731363415720/Put/seqid=0 2024-11-11T22:17:00,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741944_1120 (size=12301) 2024-11-11T22:17:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:17:00,607 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=623 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/a47ef9745d52496f85d3777d0b83919e 2024-11-11T22:17:00,616 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/A/4fbd4545c9984b0aad25585920ec3572 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4fbd4545c9984b0aad25585920ec3572 2024-11-11T22:17:00,628 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4fbd4545c9984b0aad25585920ec3572, entries=150, sequenceid=623, filesize=12.0 K 2024-11-11T22:17:00,631 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/B/7b3373198e0941c9bcb837aa87de46f5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7b3373198e0941c9bcb837aa87de46f5 2024-11-11T22:17:00,641 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7b3373198e0941c9bcb837aa87de46f5, entries=150, sequenceid=623, filesize=12.0 K 2024-11-11T22:17:00,642 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/.tmp/C/a47ef9745d52496f85d3777d0b83919e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a47ef9745d52496f85d3777d0b83919e 2024-11-11T22:17:00,648 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a47ef9745d52496f85d3777d0b83919e, entries=150, sequenceid=623, filesize=12.0 K 2024-11-11T22:17:00,656 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3f60f99bf48ef670ca729411903c0391 in 1331ms, sequenceid=623, compaction requested=true 2024-11-11T22:17:00,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3f24f20d12724335945df725acaafda1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ffe5de3d94fa4a27b581a8364b57aeb4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f7c0dcf408ef4527964b8b00ccaae1f0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/30bdd39a409442ad8e40ec8930023dfe, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/6992680497964146ab63fc2e78feb1d4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/703d29192de14a098a0ee14c3f17cdd0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e8558d86e764ca69aaed741adb9f4d0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/12fd55b8df4647b9b4ef547119f2dbd2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3a145e8c1b054f24846f9f95b890874a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/495bd65b6158492b8ce8af4e2821d92f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/dc0d51d103804e39af5dc0fc220ae846, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/80ed7e4bae454b1cb4fcd6cb8edd1688, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c0034c46457445cea6ecefbbfb0246ac, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f25270bd51e848d4b9376a4442f9da70, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/af1f37bf88594293b5a1e67524d7cf7a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b67502a2d1304a2f911cb1b285436411, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b42e506cd89741388833b9c194ccb639, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c93a8e7123b34996bbe744817152dfb7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/a287774b861b4900b185e377bfeaddca, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3170634999dc4e3f959945845bce4673, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4887b95cd8ae42148b7c47a5e076b219, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ce974c0acc744019aed963e88602d5fe, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fa00d81fd2364f3eb61ccaab9678bf38, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/119445c2ac034db781fcc8fecb344901, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f5bd41da6d6449068896ec7c7b07c8cb, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/743f522259524b3289bb8fd98d59231c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fef903132bc24ae38ef300e78fca63ac, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/91af8bd98d074c8381e148d2d7c4870d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c762a6929c5d458197055498c521f655, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4c86394dbf794b45aacb8b5fcb36b13b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/0bd7a141d1e642a7ae5c130845c8e75d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/9b93fa8140044867b5cf26add6707b66] to archive 2024-11-11T22:17:00,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:00,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3f24f20d12724335945df725acaafda1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3f24f20d12724335945df725acaafda1 2024-11-11T22:17:00,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ffe5de3d94fa4a27b581a8364b57aeb4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ffe5de3d94fa4a27b581a8364b57aeb4 2024-11-11T22:17:00,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f7c0dcf408ef4527964b8b00ccaae1f0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f7c0dcf408ef4527964b8b00ccaae1f0 2024-11-11T22:17:00,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/30bdd39a409442ad8e40ec8930023dfe to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/30bdd39a409442ad8e40ec8930023dfe 2024-11-11T22:17:00,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/6992680497964146ab63fc2e78feb1d4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/6992680497964146ab63fc2e78feb1d4 2024-11-11T22:17:00,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/703d29192de14a098a0ee14c3f17cdd0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/703d29192de14a098a0ee14c3f17cdd0 2024-11-11T22:17:00,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e8558d86e764ca69aaed741adb9f4d0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e8558d86e764ca69aaed741adb9f4d0 2024-11-11T22:17:00,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/12fd55b8df4647b9b4ef547119f2dbd2 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/12fd55b8df4647b9b4ef547119f2dbd2 2024-11-11T22:17:00,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3a145e8c1b054f24846f9f95b890874a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3a145e8c1b054f24846f9f95b890874a 2024-11-11T22:17:00,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/495bd65b6158492b8ce8af4e2821d92f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/495bd65b6158492b8ce8af4e2821d92f 2024-11-11T22:17:00,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/dc0d51d103804e39af5dc0fc220ae846 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/dc0d51d103804e39af5dc0fc220ae846 2024-11-11T22:17:00,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/80ed7e4bae454b1cb4fcd6cb8edd1688 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/80ed7e4bae454b1cb4fcd6cb8edd1688 2024-11-11T22:17:00,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c0034c46457445cea6ecefbbfb0246ac to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c0034c46457445cea6ecefbbfb0246ac 2024-11-11T22:17:00,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f25270bd51e848d4b9376a4442f9da70 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f25270bd51e848d4b9376a4442f9da70 2024-11-11T22:17:00,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/af1f37bf88594293b5a1e67524d7cf7a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/af1f37bf88594293b5a1e67524d7cf7a 2024-11-11T22:17:00,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b67502a2d1304a2f911cb1b285436411 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b67502a2d1304a2f911cb1b285436411 2024-11-11T22:17:00,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b42e506cd89741388833b9c194ccb639 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/b42e506cd89741388833b9c194ccb639 2024-11-11T22:17:00,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c93a8e7123b34996bbe744817152dfb7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c93a8e7123b34996bbe744817152dfb7 2024-11-11T22:17:00,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/a287774b861b4900b185e377bfeaddca to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/a287774b861b4900b185e377bfeaddca 2024-11-11T22:17:00,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3170634999dc4e3f959945845bce4673 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/3170634999dc4e3f959945845bce4673 2024-11-11T22:17:00,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4887b95cd8ae42148b7c47a5e076b219 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4887b95cd8ae42148b7c47a5e076b219 2024-11-11T22:17:00,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ce974c0acc744019aed963e88602d5fe to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/ce974c0acc744019aed963e88602d5fe 2024-11-11T22:17:00,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fa00d81fd2364f3eb61ccaab9678bf38 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fa00d81fd2364f3eb61ccaab9678bf38 2024-11-11T22:17:00,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/119445c2ac034db781fcc8fecb344901 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/119445c2ac034db781fcc8fecb344901 2024-11-11T22:17:00,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f5bd41da6d6449068896ec7c7b07c8cb to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f5bd41da6d6449068896ec7c7b07c8cb 2024-11-11T22:17:00,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/743f522259524b3289bb8fd98d59231c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/743f522259524b3289bb8fd98d59231c 2024-11-11T22:17:00,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fef903132bc24ae38ef300e78fca63ac to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/fef903132bc24ae38ef300e78fca63ac 2024-11-11T22:17:00,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/91af8bd98d074c8381e148d2d7c4870d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/91af8bd98d074c8381e148d2d7c4870d 2024-11-11T22:17:00,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c762a6929c5d458197055498c521f655 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/c762a6929c5d458197055498c521f655 2024-11-11T22:17:00,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4c86394dbf794b45aacb8b5fcb36b13b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4c86394dbf794b45aacb8b5fcb36b13b 2024-11-11T22:17:00,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/0bd7a141d1e642a7ae5c130845c8e75d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/0bd7a141d1e642a7ae5c130845c8e75d 2024-11-11T22:17:00,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/9b93fa8140044867b5cf26add6707b66 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/9b93fa8140044867b5cf26add6707b66 2024-11-11T22:17:00,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/aff8cba8a6fd4134b69cf212fac1e760, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/0b08798ce9ad42eca345cabcc0be6d55, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d14e2756f0646d8816b93a57c775b42, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2429849c5016433db1337668fa2eacb5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/500a0726ec63496d9207d6ff2ba0154c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/4e47121de60c41149f456bb9a327a750, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/52234c903baf447e9bde7d15ab6337b0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3737b4fca4f94f50b76c1337fa0f685e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/a652a83b161a487c86697754268b494f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcdf05b2be6e465aa8016d7ce6efa136, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/bc25b6d5dda743f6ac3c50715ee95801, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcfa020f1dc941c8ae46e7262629a2c7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/d67b5dd6c5874786abfaf58631fa351c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/8a6860f2169a4c07b3cebdc7cca8d614, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/62a0a752f080402b89da0f05997fdd08, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b0df686c180242f0b92df8b3c7f72e46, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/fe9e7926b74c4ed592770eb27fa8552a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7416a81d6f4a4edbb1cde6f9c7ae23a8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e77f83f1d5ce445d8cdccbbcf9cc171f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/1f7e4eba2f804a63b54619d973875960, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/c2eddc77657e4d8e8772e9196c8417c1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9c193a48696e4c94b9b1dfc0de638a03, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e07b626d0cc846c5beee8d37020f6267, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/eb2001fc3f744e1c89d743b8df641548, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/915bc9cdba5d4a669cbd3a55f2a4cd6b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/af29b482bd91409aa9bd5cc4f8124b20, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9187264dd8fb4bb6a15d819f81d125f0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b9d0370b5a6140bdb5da5b94d7fa7ddd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d3f0f797bbd42e69fdff111a164de28, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2cab8f8420cc45039e39a4e65b1bdb9a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/5aca413cf6914b6796af058bd2b67d42, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/348dfa30586b401db8ce05d1e5d240e7] to archive 2024-11-11T22:17:00,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:00,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/aff8cba8a6fd4134b69cf212fac1e760 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/aff8cba8a6fd4134b69cf212fac1e760 2024-11-11T22:17:00,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/0b08798ce9ad42eca345cabcc0be6d55 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/0b08798ce9ad42eca345cabcc0be6d55 2024-11-11T22:17:00,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d14e2756f0646d8816b93a57c775b42 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d14e2756f0646d8816b93a57c775b42 2024-11-11T22:17:00,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2429849c5016433db1337668fa2eacb5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2429849c5016433db1337668fa2eacb5 2024-11-11T22:17:00,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/500a0726ec63496d9207d6ff2ba0154c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/500a0726ec63496d9207d6ff2ba0154c 2024-11-11T22:17:00,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/4e47121de60c41149f456bb9a327a750 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/4e47121de60c41149f456bb9a327a750 2024-11-11T22:17:00,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/52234c903baf447e9bde7d15ab6337b0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/52234c903baf447e9bde7d15ab6337b0 2024-11-11T22:17:00,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3737b4fca4f94f50b76c1337fa0f685e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3737b4fca4f94f50b76c1337fa0f685e 2024-11-11T22:17:00,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/a652a83b161a487c86697754268b494f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/a652a83b161a487c86697754268b494f 2024-11-11T22:17:00,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcdf05b2be6e465aa8016d7ce6efa136 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcdf05b2be6e465aa8016d7ce6efa136 2024-11-11T22:17:00,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/bc25b6d5dda743f6ac3c50715ee95801 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/bc25b6d5dda743f6ac3c50715ee95801 2024-11-11T22:17:00,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcfa020f1dc941c8ae46e7262629a2c7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/dcfa020f1dc941c8ae46e7262629a2c7 2024-11-11T22:17:00,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/d67b5dd6c5874786abfaf58631fa351c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/d67b5dd6c5874786abfaf58631fa351c 2024-11-11T22:17:00,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/8a6860f2169a4c07b3cebdc7cca8d614 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/8a6860f2169a4c07b3cebdc7cca8d614 2024-11-11T22:17:00,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/62a0a752f080402b89da0f05997fdd08 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/62a0a752f080402b89da0f05997fdd08 2024-11-11T22:17:00,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b0df686c180242f0b92df8b3c7f72e46 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b0df686c180242f0b92df8b3c7f72e46 2024-11-11T22:17:00,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/fe9e7926b74c4ed592770eb27fa8552a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/fe9e7926b74c4ed592770eb27fa8552a 2024-11-11T22:17:00,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7416a81d6f4a4edbb1cde6f9c7ae23a8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7416a81d6f4a4edbb1cde6f9c7ae23a8 2024-11-11T22:17:00,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e77f83f1d5ce445d8cdccbbcf9cc171f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e77f83f1d5ce445d8cdccbbcf9cc171f 2024-11-11T22:17:00,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/1f7e4eba2f804a63b54619d973875960 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/1f7e4eba2f804a63b54619d973875960 2024-11-11T22:17:00,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/c2eddc77657e4d8e8772e9196c8417c1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/c2eddc77657e4d8e8772e9196c8417c1 2024-11-11T22:17:00,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9c193a48696e4c94b9b1dfc0de638a03 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9c193a48696e4c94b9b1dfc0de638a03 2024-11-11T22:17:00,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e07b626d0cc846c5beee8d37020f6267 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/e07b626d0cc846c5beee8d37020f6267 2024-11-11T22:17:00,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/eb2001fc3f744e1c89d743b8df641548 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/eb2001fc3f744e1c89d743b8df641548 2024-11-11T22:17:00,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/915bc9cdba5d4a669cbd3a55f2a4cd6b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/915bc9cdba5d4a669cbd3a55f2a4cd6b 2024-11-11T22:17:00,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/af29b482bd91409aa9bd5cc4f8124b20 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/af29b482bd91409aa9bd5cc4f8124b20 2024-11-11T22:17:00,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9187264dd8fb4bb6a15d819f81d125f0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/9187264dd8fb4bb6a15d819f81d125f0 2024-11-11T22:17:00,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b9d0370b5a6140bdb5da5b94d7fa7ddd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/b9d0370b5a6140bdb5da5b94d7fa7ddd 2024-11-11T22:17:00,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d3f0f797bbd42e69fdff111a164de28 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/3d3f0f797bbd42e69fdff111a164de28 2024-11-11T22:17:00,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2cab8f8420cc45039e39a4e65b1bdb9a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/2cab8f8420cc45039e39a4e65b1bdb9a 2024-11-11T22:17:00,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/5aca413cf6914b6796af058bd2b67d42 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/5aca413cf6914b6796af058bd2b67d42 2024-11-11T22:17:00,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/348dfa30586b401db8ce05d1e5d240e7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/348dfa30586b401db8ce05d1e5d240e7 2024-11-11T22:17:01,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9aeba5a1914343c89aa1d31b2e9fd1c2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6dff705f4509487e8c79e655153f39cc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/61a24b7b34e945e0aa6ca9282e15db66, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/606820d66436480e8f574350c09b19d1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d753f4548f5d4a31bae4fe29146ac04a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e1f7b7c14038432f89b252a01201c965, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/1a0566565cf44598b6727fefb6f942ad, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/973073218b0a4da0aeef97f022d78ce4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7f2eb2cdb16b4d40a8301e9c4d3ab9b5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a31a8745da9b4bfea895763370184750, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d632aa6ba333411ab1b3cee554888eb4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/38ca90ee19a948ef92688cc96423e441, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/0f2f4ab68e2a4552b9f6328628275a61, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/91dd2d5a0939434a8e2804c413aac3f5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/80187ee7a54740a390838b71a1d60fbd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9a0b3ec157624085bf92279f8de8e4e8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/71f3385d115b48df8d429e1d1ccce27a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/4d9d5a2aaaf947548395c96db5938e6b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6c54fe0e006546ebad932f83651136fd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/556e7620999341e488d5536cf09ad74f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9f6bb3d2751c4e30adfd7278331ac97d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/f4b8785092e94297a7eb0807fd930bd6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/67489fff8a054f299da3a16a10f37979, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7e1b15f3e0ff40059d504b7e4d7c5be5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/17edcb57047a4ed281759dc350e4da01, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/dbfd73791b9945749423e60817d8c7b9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/caf08db6f2964452a4996b991bb59daa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9b75d36da50048a39d03ec2747a10ec8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/18938ed185af44dda444e44bfa2d6794, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e33a4091d6fa4150900fea4f0748f1e9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/fd94ac1fb80843ae9823162a15eaa4bd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e8f167efc7ca4f01ba6a10e3e60f926a] to archive 2024-11-11T22:17:01,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:01,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9aeba5a1914343c89aa1d31b2e9fd1c2 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9aeba5a1914343c89aa1d31b2e9fd1c2 2024-11-11T22:17:01,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6dff705f4509487e8c79e655153f39cc to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6dff705f4509487e8c79e655153f39cc 2024-11-11T22:17:01,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/61a24b7b34e945e0aa6ca9282e15db66 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/61a24b7b34e945e0aa6ca9282e15db66 2024-11-11T22:17:01,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/606820d66436480e8f574350c09b19d1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/606820d66436480e8f574350c09b19d1 2024-11-11T22:17:01,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d753f4548f5d4a31bae4fe29146ac04a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d753f4548f5d4a31bae4fe29146ac04a 2024-11-11T22:17:01,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e1f7b7c14038432f89b252a01201c965 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e1f7b7c14038432f89b252a01201c965 2024-11-11T22:17:01,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/1a0566565cf44598b6727fefb6f942ad to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/1a0566565cf44598b6727fefb6f942ad 2024-11-11T22:17:01,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/973073218b0a4da0aeef97f022d78ce4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/973073218b0a4da0aeef97f022d78ce4 2024-11-11T22:17:01,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7f2eb2cdb16b4d40a8301e9c4d3ab9b5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7f2eb2cdb16b4d40a8301e9c4d3ab9b5 2024-11-11T22:17:01,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a31a8745da9b4bfea895763370184750 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a31a8745da9b4bfea895763370184750 2024-11-11T22:17:01,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d632aa6ba333411ab1b3cee554888eb4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/d632aa6ba333411ab1b3cee554888eb4 2024-11-11T22:17:01,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/38ca90ee19a948ef92688cc96423e441 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/38ca90ee19a948ef92688cc96423e441 2024-11-11T22:17:01,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/0f2f4ab68e2a4552b9f6328628275a61 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/0f2f4ab68e2a4552b9f6328628275a61 2024-11-11T22:17:01,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/91dd2d5a0939434a8e2804c413aac3f5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/91dd2d5a0939434a8e2804c413aac3f5 2024-11-11T22:17:01,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/80187ee7a54740a390838b71a1d60fbd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/80187ee7a54740a390838b71a1d60fbd 2024-11-11T22:17:01,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9a0b3ec157624085bf92279f8de8e4e8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9a0b3ec157624085bf92279f8de8e4e8 2024-11-11T22:17:01,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/71f3385d115b48df8d429e1d1ccce27a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/71f3385d115b48df8d429e1d1ccce27a 2024-11-11T22:17:01,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/4d9d5a2aaaf947548395c96db5938e6b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/4d9d5a2aaaf947548395c96db5938e6b 2024-11-11T22:17:01,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6c54fe0e006546ebad932f83651136fd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/6c54fe0e006546ebad932f83651136fd 2024-11-11T22:17:01,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/556e7620999341e488d5536cf09ad74f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/556e7620999341e488d5536cf09ad74f 2024-11-11T22:17:01,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9f6bb3d2751c4e30adfd7278331ac97d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9f6bb3d2751c4e30adfd7278331ac97d 2024-11-11T22:17:01,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/f4b8785092e94297a7eb0807fd930bd6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/f4b8785092e94297a7eb0807fd930bd6 2024-11-11T22:17:01,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/67489fff8a054f299da3a16a10f37979 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/67489fff8a054f299da3a16a10f37979 2024-11-11T22:17:01,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7e1b15f3e0ff40059d504b7e4d7c5be5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/7e1b15f3e0ff40059d504b7e4d7c5be5 2024-11-11T22:17:01,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/17edcb57047a4ed281759dc350e4da01 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/17edcb57047a4ed281759dc350e4da01 2024-11-11T22:17:01,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/dbfd73791b9945749423e60817d8c7b9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/dbfd73791b9945749423e60817d8c7b9 2024-11-11T22:17:01,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/caf08db6f2964452a4996b991bb59daa to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/caf08db6f2964452a4996b991bb59daa 2024-11-11T22:17:01,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9b75d36da50048a39d03ec2747a10ec8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/9b75d36da50048a39d03ec2747a10ec8 2024-11-11T22:17:01,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/18938ed185af44dda444e44bfa2d6794 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/18938ed185af44dda444e44bfa2d6794 2024-11-11T22:17:01,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e33a4091d6fa4150900fea4f0748f1e9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e33a4091d6fa4150900fea4f0748f1e9 2024-11-11T22:17:01,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/fd94ac1fb80843ae9823162a15eaa4bd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/fd94ac1fb80843ae9823162a15eaa4bd 2024-11-11T22:17:01,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e8f167efc7ca4f01ba6a10e3e60f926a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/e8f167efc7ca4f01ba6a10e3e60f926a 2024-11-11T22:17:01,204 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/recovered.edits/626.seqid, newMaxSeqId=626, maxSeqId=1 2024-11-11T22:17:01,206 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. 2024-11-11T22:17:01,206 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 3f60f99bf48ef670ca729411903c0391: Waiting for close lock at 1731363419325Running coprocessor pre-close hooks at 1731363419325Disabling compacts and flushes for region at 1731363419325Disabling writes for close at 1731363419325Obtaining lock to block concurrent updates at 1731363419325Preparing flush snapshotting stores in 3f60f99bf48ef670ca729411903c0391 at 1731363419325Finished memstore snapshotting TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731363419326 (+1 ms)Flushing stores of TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391. at 1731363419326Flushing 3f60f99bf48ef670ca729411903c0391/A: creating writer at 1731363419326Flushing 3f60f99bf48ef670ca729411903c0391/A: appending metadata at 1731363419330 (+4 ms)Flushing 3f60f99bf48ef670ca729411903c0391/A: closing flushed file at 1731363419331 (+1 ms)Flushing 3f60f99bf48ef670ca729411903c0391/B: creating writer at 1731363419769 (+438 ms)Flushing 3f60f99bf48ef670ca729411903c0391/B: appending metadata at 1731363419773 (+4 ms)Flushing 3f60f99bf48ef670ca729411903c0391/B: closing flushed file at 1731363419773Flushing 3f60f99bf48ef670ca729411903c0391/C: creating writer at 1731363420198 (+425 ms)Flushing 3f60f99bf48ef670ca729411903c0391/C: appending metadata at 1731363420201 (+3 ms)Flushing 3f60f99bf48ef670ca729411903c0391/C: closing flushed file at 1731363420201Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@152345cc: reopening flushed file at 1731363420615 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22a73bc1: reopening flushed file at 1731363420628 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c19374b: reopening flushed file at 1731363420641 (+13 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3f60f99bf48ef670ca729411903c0391 in 1331ms, sequenceid=623, compaction requested=true at 1731363420656 (+15 ms)Writing region close event to WAL at 1731363421200 (+544 ms)Running coprocessor post-close hooks at 1731363421205 (+5 ms)Closed at 1731363421206 (+1 ms) 2024-11-11T22:17:01,209 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 3f60f99bf48ef670ca729411903c0391 2024-11-11T22:17:01,210 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3f60f99bf48ef670ca729411903c0391, regionState=CLOSED 2024-11-11T22:17:01,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:17:01,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T22:17:01,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 3f60f99bf48ef670ca729411903c0391, server=8d82b89dd27d,36911,1731363387997 in 2.0630 sec 2024-11-11T22:17:01,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-11T22:17:01,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3f60f99bf48ef670ca729411903c0391, UNASSIGN in 2.0730 sec 2024-11-11T22:17:01,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T22:17:01,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0890 sec 2024-11-11T22:17:01,253 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363421252"}]},"ts":"1731363421252"} 2024-11-11T22:17:01,261 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:17:01,261 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:17:01,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1290 sec 2024-11-11T22:17:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-11T22:17:01,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:01,308 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:17:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:01,315 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T22:17:01,317 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:01,331 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391 2024-11-11T22:17:01,369 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/recovered.edits] 2024-11-11T22:17:01,406 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4fbd4545c9984b0aad25585920ec3572 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/4fbd4545c9984b0aad25585920ec3572 2024-11-11T22:17:01,417 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e03e75d551b40a988e4f5e46fd32394 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/5e03e75d551b40a988e4f5e46fd32394 2024-11-11T22:17:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T22:17:01,437 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/e1bf747380ec4364a4e1c5f58bbe6a5b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/e1bf747380ec4364a4e1c5f58bbe6a5b 2024-11-11T22:17:01,439 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f899e63ddc474afaae6336e6298a847b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/A/f899e63ddc474afaae6336e6298a847b 2024-11-11T22:17:01,450 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/71fdea5667ca4e68a17ebe9f40b8b087 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/71fdea5667ca4e68a17ebe9f40b8b087 2024-11-11T22:17:01,454 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7b05a9c7a38342aab6faeef328794cc3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7b05a9c7a38342aab6faeef328794cc3 2024-11-11T22:17:01,457 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7b3373198e0941c9bcb837aa87de46f5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7b3373198e0941c9bcb837aa87de46f5 2024-11-11T22:17:01,460 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7e86760b562f4a4ba76533cd2a8651d7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/B/7e86760b562f4a4ba76533cd2a8651d7 2024-11-11T22:17:01,469 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/353012ef37f444df86558c08d30b6f41 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/353012ef37f444df86558c08d30b6f41 2024-11-11T22:17:01,481 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/5c5c1e2e5b914ca8859c6727713a278a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/5c5c1e2e5b914ca8859c6727713a278a 2024-11-11T22:17:01,484 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a47ef9745d52496f85d3777d0b83919e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/a47ef9745d52496f85d3777d0b83919e 2024-11-11T22:17:01,488 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/be6e2f0608c8469ba1ec7a909a29f569 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/C/be6e2f0608c8469ba1ec7a909a29f569 2024-11-11T22:17:01,495 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/recovered.edits/626.seqid to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391/recovered.edits/626.seqid 2024-11-11T22:17:01,497 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/3f60f99bf48ef670ca729411903c0391 2024-11-11T22:17:01,497 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:17:01,519 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:01,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T22:17:01,542 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:17:01,608 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:17:01,613 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:01,613 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:17:01,614 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731363421613"}]},"ts":"9223372036854775807"} 2024-11-11T22:17:01,630 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:17:01,630 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3f60f99bf48ef670ca729411903c0391, NAME => 'TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:17:01,631 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:17:01,631 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731363421631"}]},"ts":"9223372036854775807"} 2024-11-11T22:17:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T22:17:01,639 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:17:01,642 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:01,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 333 msec 2024-11-11T22:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-11T22:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:17:01,941 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:01,955 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;8d82b89dd27d:36911-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1014836282_22 at /127.0.0.1:47810 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=640 (was 536) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1596 (was 1756) 2024-11-11T22:17:01,967 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=640, ProcessCount=11, AvailableMemoryMB=1595 2024-11-11T22:17:01,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:17:01,970 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:17:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:01,973 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:17:01,973 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:01,973 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-11T22:17:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:17:01,974 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:17:01,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741945_1121 (size=1061) 2024-11-11T22:17:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:17:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:17:02,384 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e 2024-11-11T22:17:02,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741946_1122 (size=53) 2024-11-11T22:17:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:17:02,790 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:17:02,790 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4c17d89df59d2cd5ce9ff2614859e6e4, disabling compactions & flushes 2024-11-11T22:17:02,791 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:02,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:02,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. after waiting 0 ms 2024-11-11T22:17:02,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:02,791 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:02,791 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c17d89df59d2cd5ce9ff2614859e6e4: Waiting for close lock at 1731363422790Disabling compacts and flushes for region at 1731363422790Disabling writes for close at 1731363422791 (+1 ms)Writing region close event to WAL at 1731363422791Closed at 1731363422791 2024-11-11T22:17:02,792 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:17:02,793 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363422792"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363422792"}]},"ts":"1731363422792"} 2024-11-11T22:17:02,795 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:17:02,797 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:17:02,797 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363422797"}]},"ts":"1731363422797"} 2024-11-11T22:17:02,800 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:17:02,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, ASSIGN}] 2024-11-11T22:17:02,802 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, ASSIGN 2024-11-11T22:17:02,804 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, ASSIGN; state=OFFLINE, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=false 2024-11-11T22:17:02,955 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=4c17d89df59d2cd5ce9ff2614859e6e4, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:02,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, ASSIGN because future has completed 2024-11-11T22:17:02,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:17:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:17:03,114 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:03,114 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:17:03,114 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,114 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:17:03,114 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,114 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,116 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,117 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:03,118 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c17d89df59d2cd5ce9ff2614859e6e4 columnFamilyName A 2024-11-11T22:17:03,118 DEBUG [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:03,118 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(327): Store=4c17d89df59d2cd5ce9ff2614859e6e4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:03,119 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,120 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:03,120 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c17d89df59d2cd5ce9ff2614859e6e4 columnFamilyName B 2024-11-11T22:17:03,120 DEBUG [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:03,121 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(327): Store=4c17d89df59d2cd5ce9ff2614859e6e4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:03,121 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,122 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:03,122 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c17d89df59d2cd5ce9ff2614859e6e4 columnFamilyName C 2024-11-11T22:17:03,122 DEBUG [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:03,123 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(327): Store=4c17d89df59d2cd5ce9ff2614859e6e4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:03,123 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:03,123 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,124 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,124 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,125 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,125 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,126 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:17:03,127 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,129 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:17:03,130 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 4c17d89df59d2cd5ce9ff2614859e6e4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70889802, jitterRate=0.05634036660194397}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:17:03,130 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:03,131 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 4c17d89df59d2cd5ce9ff2614859e6e4: Running coprocessor pre-open hook at 1731363423114Writing region info on filesystem at 1731363423114Initializing all the Stores at 1731363423115 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363423115Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363423116 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363423116Cleaning up temporary data from old regions at 1731363423125 (+9 ms)Running coprocessor post-open hooks at 1731363423130 (+5 ms)Region opened successfully at 1731363423131 (+1 ms) 2024-11-11T22:17:03,132 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., pid=36, masterSystemTime=1731363423110 2024-11-11T22:17:03,134 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:03,135 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:03,135 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=4c17d89df59d2cd5ce9ff2614859e6e4, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:03,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:17:03,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T22:17:03,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 in 179 msec 2024-11-11T22:17:03,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T22:17:03,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, ASSIGN in 338 msec 2024-11-11T22:17:03,142 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:17:03,142 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363423142"}]},"ts":"1731363423142"} 2024-11-11T22:17:03,144 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:17:03,145 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:17:03,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-11T22:17:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T22:17:04,124 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:04,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2bb5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:04,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:04,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:04,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40437203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:04,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:04,140 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d1de7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:04,149 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:04,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:04,154 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:17:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,41919,1731363387169 2024-11-11T22:17:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f9f9fc2 2024-11-11T22:17:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:17:04,158 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:17:04,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:17:04,164 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:17:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:04,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741947_1123 (size=1097) 2024-11-11T22:17:04,602 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T22:17:04,603 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T22:17:04,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:17:04,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, REOPEN/MOVE}] 2024-11-11T22:17:04,618 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, REOPEN/MOVE 2024-11-11T22:17:04,619 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=4c17d89df59d2cd5ce9ff2614859e6e4, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:04,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, REOPEN/MOVE because future has completed 2024-11-11T22:17:04,622 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:17:04,622 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:17:04,775 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:04,775 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:17:04,775 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 4c17d89df59d2cd5ce9ff2614859e6e4, disabling compactions & flushes 2024-11-11T22:17:04,775 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:04,775 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:04,775 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. after waiting 0 ms 2024-11-11T22:17:04,775 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:04,779 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T22:17:04,780 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:04,780 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 4c17d89df59d2cd5ce9ff2614859e6e4: Waiting for close lock at 1731363424775Running coprocessor pre-close hooks at 1731363424775Disabling compacts and flushes for region at 1731363424775Disabling writes for close at 1731363424775Writing region close event to WAL at 1731363424776 (+1 ms)Running coprocessor post-close hooks at 1731363424780 (+4 ms)Closed at 1731363424780 2024-11-11T22:17:04,780 WARN [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 4c17d89df59d2cd5ce9ff2614859e6e4 to self. 2024-11-11T22:17:04,782 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:04,783 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=4c17d89df59d2cd5ce9ff2614859e6e4, regionState=CLOSED 2024-11-11T22:17:04,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:17:04,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-11T22:17:04,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 in 164 msec 2024-11-11T22:17:04,789 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, REOPEN/MOVE; state=CLOSED, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=true 2024-11-11T22:17:04,940 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=4c17d89df59d2cd5ce9ff2614859e6e4, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:04,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, REOPEN/MOVE because future has completed 2024-11-11T22:17:04,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:17:05,097 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,097 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:17:05,098 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,098 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:17:05,098 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,098 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,100 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,101 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:05,108 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c17d89df59d2cd5ce9ff2614859e6e4 columnFamilyName A 2024-11-11T22:17:05,113 DEBUG [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:05,113 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(327): Store=4c17d89df59d2cd5ce9ff2614859e6e4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:05,114 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,117 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:05,117 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c17d89df59d2cd5ce9ff2614859e6e4 columnFamilyName B 2024-11-11T22:17:05,117 DEBUG [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:05,118 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(327): Store=4c17d89df59d2cd5ce9ff2614859e6e4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:05,118 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,119 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:05,120 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c17d89df59d2cd5ce9ff2614859e6e4 columnFamilyName C 2024-11-11T22:17:05,120 DEBUG [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:05,121 INFO [StoreOpener-4c17d89df59d2cd5ce9ff2614859e6e4-1 {}] regionserver.HStore(327): Store=4c17d89df59d2cd5ce9ff2614859e6e4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:05,121 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,121 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,122 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,124 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,126 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,126 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,127 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:17:05,128 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,130 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 4c17d89df59d2cd5ce9ff2614859e6e4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61729026, jitterRate=-0.08016583323478699}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:17:05,130 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,132 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 4c17d89df59d2cd5ce9ff2614859e6e4: Running coprocessor pre-open hook at 1731363425098Writing region info on filesystem at 1731363425098Initializing all the Stores at 1731363425099 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363425099Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363425099Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363425099Cleaning up temporary data from old regions at 1731363425126 (+27 ms)Running coprocessor post-open hooks at 1731363425130 (+4 ms)Region opened successfully at 1731363425132 (+2 ms) 2024-11-11T22:17:05,133 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., pid=41, masterSystemTime=1731363425094 2024-11-11T22:17:05,136 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,136 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,137 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=4c17d89df59d2cd5ce9ff2614859e6e4, regionState=OPEN, openSeqNum=5, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:17:05,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-11T22:17:05,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 in 198 msec 2024-11-11T22:17:05,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T22:17:05,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, REOPEN/MOVE in 528 msec 2024-11-11T22:17:05,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-11T22:17:05,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 543 msec 2024-11-11T22:17:05,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 991 msec 2024-11-11T22:17:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-11T22:17:05,166 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:05,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e01494b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,169 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72fb1aec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,172 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541d4a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28944b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,185 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f05b7cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20292cf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a95d199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,196 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bb3c41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,198 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4249e7f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e51468f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,207 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2055a7d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2c54ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1fcab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff12e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,215 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,217 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f304f74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,220 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d6a31d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,221 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebedfb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17cb6028, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97fb45e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,230 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ba4bd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,232 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8f24a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c4632d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,239 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe1b97a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,242 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f74fba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eae57b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:05,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:05,250 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff6088f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:05,255 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:05,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be3d499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:05,261 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,265 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,266 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,279 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,289 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,291 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,293 DEBUG [Thread-639 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,293 DEBUG [Thread-639 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:17:05,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:05,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:05,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:05,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:05,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:05,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:05,305 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,309 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,334 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,335 DEBUG [Thread-632 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,335 DEBUG [Thread-632 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,344 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T22:17:05,347 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:17:05,348 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:05,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:05,365 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:05,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:17:05,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363425372, max timestamp=1731363425407Num uniques 859; 2024-11-11T22:17:05,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:05,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363425372, max timestamp=1731363425407Num uniques 859; 2024-11-11T22:17:05,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:05,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363425372, max timestamp=1731363425407Num uniques 859; 2024-11-11T22:17:05,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:05,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e33a71b022b444c6aed5f7bae998e330_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363425289/Put/seqid=0 2024-11-11T22:17:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:17:05,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741948_1124 (size=17034) 2024-11-11T22:17:05,495 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:05,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e33a71b022b444c6aed5f7bae998e330_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e33a71b022b444c6aed5f7bae998e330_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:05,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:05,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:05,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,504 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/f38349a882324dbf927adb62c307849c, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:05,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/f38349a882324dbf927adb62c307849c is 175, key is test_row_0/A:col10/1731363425289/Put/seqid=0 2024-11-11T22:17:05,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363485480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363485481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731363425473, max timestamp=1731363425502Num uniques 1018; 2024-11-11T22:17:05,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T22:17:05,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731363425473, max timestamp=1731363425502Num uniques 1018; 2024-11-11T22:17:05,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731363425473, max timestamp=1731363425502Num uniques 1018; 2024-11-11T22:17:05,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T22:17:05,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T22:17:05,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741949_1125 (size=48139) 2024-11-11T22:17:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363485616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363485620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363485622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363485624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363485628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:05,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:05,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:17:05,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363485742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363485742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363485742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363485842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363485842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,954 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/f38349a882324dbf927adb62c307849c 2024-11-11T22:17:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363485960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363485960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363485960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:05,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:05,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:17:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:05,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/5a8fd0c0b27c48b8b8edd67c190f6b59 is 50, key is test_row_0/B:col10/1731363425289/Put/seqid=0 2024-11-11T22:17:06,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741950_1126 (size=12001) 2024-11-11T22:17:06,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/5a8fd0c0b27c48b8b8edd67c190f6b59 2024-11-11T22:17:06,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5a4aba194fa04eaf86d460819b9bf550 is 50, key is test_row_0/C:col10/1731363425289/Put/seqid=0 2024-11-11T22:17:06,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741951_1127 (size=12001) 2024-11-11T22:17:06,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5a4aba194fa04eaf86d460819b9bf550 2024-11-11T22:17:06,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363486160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363486161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/f38349a882324dbf927adb62c307849c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f38349a882324dbf927adb62c307849c 2024-11-11T22:17:06,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f38349a882324dbf927adb62c307849c, entries=250, sequenceid=14, filesize=47.0 K 2024-11-11T22:17:06,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/5a8fd0c0b27c48b8b8edd67c190f6b59 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/5a8fd0c0b27c48b8b8edd67c190f6b59 2024-11-11T22:17:06,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/5a8fd0c0b27c48b8b8edd67c190f6b59, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T22:17:06,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5a4aba194fa04eaf86d460819b9bf550 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5a4aba194fa04eaf86d460819b9bf550 2024-11-11T22:17:06,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5a4aba194fa04eaf86d460819b9bf550, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T22:17:06,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 895ms, sequenceid=14, compaction requested=false 2024-11-11T22:17:06,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:06,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-11-11T22:17:06,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:06,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:06,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:06,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:06,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:06,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:06,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:06,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:06,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112026f68fc0f74fa9987f558134fb4098_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363425625/Put/seqid=0 2024-11-11T22:17:06,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741952_1128 (size=14594) 2024-11-11T22:17:06,313 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:06,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112026f68fc0f74fa9987f558134fb4098_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112026f68fc0f74fa9987f558134fb4098_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:06,321 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d694cc7bb8f844449cef48d94483f7b6, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:06,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d694cc7bb8f844449cef48d94483f7b6 is 175, key is test_row_0/A:col10/1731363425625/Put/seqid=0 2024-11-11T22:17:06,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741953_1129 (size=39549) 2024-11-11T22:17:06,334 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d694cc7bb8f844449cef48d94483f7b6 2024-11-11T22:17:06,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363486319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/1d9cd0e9dd374370b8ee085e16adf210 is 50, key is test_row_0/B:col10/1731363425625/Put/seqid=0 2024-11-11T22:17:06,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363486340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363486347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741954_1130 (size=12001) 2024-11-11T22:17:06,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/1d9cd0e9dd374370b8ee085e16adf210 2024-11-11T22:17:06,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/2324bbcad6764b3b8bee4e6de9cc2001 is 50, key is test_row_0/C:col10/1731363425625/Put/seqid=0 2024-11-11T22:17:06,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741955_1131 (size=12001) 2024-11-11T22:17:06,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/2324bbcad6764b3b8bee4e6de9cc2001 2024-11-11T22:17:06,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d694cc7bb8f844449cef48d94483f7b6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d694cc7bb8f844449cef48d94483f7b6 2024-11-11T22:17:06,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d694cc7bb8f844449cef48d94483f7b6, entries=200, sequenceid=68, filesize=38.6 K 2024-11-11T22:17:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/1d9cd0e9dd374370b8ee085e16adf210 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/1d9cd0e9dd374370b8ee085e16adf210 2024-11-11T22:17:06,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/1d9cd0e9dd374370b8ee085e16adf210, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T22:17:06,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/2324bbcad6764b3b8bee4e6de9cc2001 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2324bbcad6764b3b8bee4e6de9cc2001 2024-11-11T22:17:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363486449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363486460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2324bbcad6764b3b8bee4e6de9cc2001, entries=150, sequenceid=68, filesize=11.7 K 2024-11-11T22:17:06,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=46.96 KB/48090 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 184ms, sequenceid=68, compaction requested=false 2024-11-11T22:17:06,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:06,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T22:17:06,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:06,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:06,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:06,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:06,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:06,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:17:06,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119b2a4737d3af4750999f8c2720a8efa6_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363426295/Put/seqid=0 2024-11-11T22:17:06,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741956_1132 (size=14594) 2024-11-11T22:17:06,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363426596, max timestamp=1731363426628Num uniques 859; 2024-11-11T22:17:06,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:06,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363426596, max timestamp=1731363426628Num uniques 859; 2024-11-11T22:17:06,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363426596, max timestamp=1731363426628Num uniques 859; 2024-11-11T22:17:06,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:06,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:06,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363486702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363486704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363486722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363486722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363486728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,771 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:17:06,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363486831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363486831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363486841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363486841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363486841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:06,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:06,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:06,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:06,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:06,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:06,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:06,940 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119b2a4737d3af4750999f8c2720a8efa6_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119b2a4737d3af4750999f8c2720a8efa6_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:06,944 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/be6e7feb14a343e89fce07c8f2ac91cd, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:06,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/be6e7feb14a343e89fce07c8f2ac91cd is 175, key is test_row_0/A:col10/1731363426295/Put/seqid=0 2024-11-11T22:17:06,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741957_1133 (size=39549) 2024-11-11T22:17:06,977 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/be6e7feb14a343e89fce07c8f2ac91cd 2024-11-11T22:17:07,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/eb990e229d114d73abac3670eee83faf is 50, key is test_row_0/B:col10/1731363426295/Put/seqid=0 2024-11-11T22:17:07,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363487093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363487095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:07,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363487097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363487097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363487098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741958_1134 (size=12001) 2024-11-11T22:17:07,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/eb990e229d114d73abac3670eee83faf 2024-11-11T22:17:07,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/96937dfdfbec438b832f1a0a10153fd2 is 50, key is test_row_0/C:col10/1731363426295/Put/seqid=0 2024-11-11T22:17:07,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741959_1135 (size=12001) 2024-11-11T22:17:07,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363487410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363487410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363487410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363487411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363487420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:17:07,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:07,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/96937dfdfbec438b832f1a0a10153fd2 2024-11-11T22:17:07,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/be6e7feb14a343e89fce07c8f2ac91cd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/be6e7feb14a343e89fce07c8f2ac91cd 2024-11-11T22:17:07,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/be6e7feb14a343e89fce07c8f2ac91cd, entries=200, sequenceid=79, filesize=38.6 K 2024-11-11T22:17:07,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/eb990e229d114d73abac3670eee83faf as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/eb990e229d114d73abac3670eee83faf 2024-11-11T22:17:07,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:07,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/eb990e229d114d73abac3670eee83faf, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:17:07,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/96937dfdfbec438b832f1a0a10153fd2 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/96937dfdfbec438b832f1a0a10153fd2 2024-11-11T22:17:07,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/96937dfdfbec438b832f1a0a10153fd2, entries=150, sequenceid=79, filesize=11.7 K 2024-11-11T22:17:07,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1356ms, sequenceid=79, compaction requested=true 2024-11-11T22:17:07,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:07,830 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:17:07,830 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:07,832 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:07,832 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/A is initiating minor compaction (all files) 2024-11-11T22:17:07,832 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/A in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,832 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f38349a882324dbf927adb62c307849c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d694cc7bb8f844449cef48d94483f7b6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/be6e7feb14a343e89fce07c8f2ac91cd] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=124.3 K 2024-11-11T22:17:07,833 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,833 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f38349a882324dbf927adb62c307849c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d694cc7bb8f844449cef48d94483f7b6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/be6e7feb14a343e89fce07c8f2ac91cd] 2024-11-11T22:17:07,837 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f38349a882324dbf927adb62c307849c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363425279 2024-11-11T22:17:07,838 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d694cc7bb8f844449cef48d94483f7b6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731363425613 2024-11-11T22:17:07,838 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:07,838 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/B is initiating minor compaction (all files) 2024-11-11T22:17:07,838 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/B in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,838 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/5a8fd0c0b27c48b8b8edd67c190f6b59, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/1d9cd0e9dd374370b8ee085e16adf210, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/eb990e229d114d73abac3670eee83faf] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=35.2 K 2024-11-11T22:17:07,839 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting be6e7feb14a343e89fce07c8f2ac91cd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363426295 2024-11-11T22:17:07,839 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a8fd0c0b27c48b8b8edd67c190f6b59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363425283 2024-11-11T22:17:07,840 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d9cd0e9dd374370b8ee085e16adf210, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731363425613 2024-11-11T22:17:07,841 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb990e229d114d73abac3670eee83faf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363426295 2024-11-11T22:17:07,865 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:07,871 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#B#compaction#118 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:07,873 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/c7617e2c90a8475cafe87996f6ff944f is 50, key is test_row_0/B:col10/1731363426295/Put/seqid=0 2024-11-11T22:17:07,880 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111193fc8d6d061d4475ab2bc6f4639aae1d_4c17d89df59d2cd5ce9ff2614859e6e4 store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:07,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T22:17:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,899 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T22:17:07,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:07,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:07,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:07,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:07,899 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111193fc8d6d061d4475ab2bc6f4639aae1d_4c17d89df59d2cd5ce9ff2614859e6e4, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:07,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:07,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:07,900 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111193fc8d6d061d4475ab2bc6f4639aae1d_4c17d89df59d2cd5ce9ff2614859e6e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:07,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741960_1136 (size=12104) 2024-11-11T22:17:07,943 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/c7617e2c90a8475cafe87996f6ff944f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/c7617e2c90a8475cafe87996f6ff944f 2024-11-11T22:17:07,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:07,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741961_1137 (size=4469) 2024-11-11T22:17:07,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111892989b19661488c81ecee862a16745d_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363426720/Put/seqid=0 2024-11-11T22:17:07,958 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/B of 4c17d89df59d2cd5ce9ff2614859e6e4 into c7617e2c90a8475cafe87996f6ff944f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:07,958 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:07,958 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/B, priority=13, startTime=1731363427830; duration=0sec 2024-11-11T22:17:07,958 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:07,958 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:B 2024-11-11T22:17:07,958 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:07,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:07,961 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:07,961 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/C is initiating minor compaction (all files) 2024-11-11T22:17:07,961 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/C in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:07,961 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5a4aba194fa04eaf86d460819b9bf550, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2324bbcad6764b3b8bee4e6de9cc2001, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/96937dfdfbec438b832f1a0a10153fd2] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=35.2 K 2024-11-11T22:17:07,962 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a4aba194fa04eaf86d460819b9bf550, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363425283 2024-11-11T22:17:07,962 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2324bbcad6764b3b8bee4e6de9cc2001, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731363425613 2024-11-11T22:17:07,963 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96937dfdfbec438b832f1a0a10153fd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363426295 2024-11-11T22:17:07,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741962_1138 (size=12154) 2024-11-11T22:17:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:07,986 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111892989b19661488c81ecee862a16745d_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111892989b19661488c81ecee862a16745d_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:07,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2ee0416cb224f07a8d90761053f719f, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:07,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363487954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2ee0416cb224f07a8d90761053f719f is 175, key is test_row_0/A:col10/1731363426720/Put/seqid=0 2024-11-11T22:17:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363487955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,989 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#C#compaction#120 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363487960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:07,992 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/072f63adb9be4efd8d0f826c2ff7796e is 50, key is test_row_0/C:col10/1731363426295/Put/seqid=0 2024-11-11T22:17:08,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363487984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363487989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741963_1139 (size=30955) 2024-11-11T22:17:08,033 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2ee0416cb224f07a8d90761053f719f 2024-11-11T22:17:08,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741964_1140 (size=12104) 2024-11-11T22:17:08,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/4db1a001b5cc4824be6a73de4e925e4f is 50, key is test_row_0/B:col10/1731363426720/Put/seqid=0 2024-11-11T22:17:08,068 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/072f63adb9be4efd8d0f826c2ff7796e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/072f63adb9be4efd8d0f826c2ff7796e 2024-11-11T22:17:08,078 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/C of 4c17d89df59d2cd5ce9ff2614859e6e4 into 072f63adb9be4efd8d0f826c2ff7796e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:08,078 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:08,078 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/C, priority=13, startTime=1731363427830; duration=0sec 2024-11-11T22:17:08,078 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:08,079 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:C 2024-11-11T22:17:08,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363488099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363488100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363488100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363488109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363488109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741965_1141 (size=12001) 2024-11-11T22:17:08,132 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/4db1a001b5cc4824be6a73de4e925e4f 2024-11-11T22:17:08,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/f0e9470c8bf140fa8abfe2fce062e844 is 50, key is test_row_0/C:col10/1731363426720/Put/seqid=0 2024-11-11T22:17:08,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741966_1142 (size=12001) 2024-11-11T22:17:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363488310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363488310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363488310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363488320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363488320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,357 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#A#compaction#117 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:08,358 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/0723b8fb54374f7ca9533c7baae089e3 is 175, key is test_row_0/A:col10/1731363426295/Put/seqid=0 2024-11-11T22:17:08,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741967_1143 (size=31058) 2024-11-11T22:17:08,423 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/0723b8fb54374f7ca9533c7baae089e3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0723b8fb54374f7ca9533c7baae089e3 2024-11-11T22:17:08,435 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/A of 4c17d89df59d2cd5ce9ff2614859e6e4 into 0723b8fb54374f7ca9533c7baae089e3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:08,435 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:08,435 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/A, priority=13, startTime=1731363427829; duration=0sec 2024-11-11T22:17:08,435 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:08,435 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:A 2024-11-11T22:17:08,633 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/f0e9470c8bf140fa8abfe2fce062e844 2024-11-11T22:17:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363488631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363488631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363488630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363488640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:08,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363488644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:08,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2ee0416cb224f07a8d90761053f719f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2ee0416cb224f07a8d90761053f719f 2024-11-11T22:17:08,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:08,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:08,674 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2ee0416cb224f07a8d90761053f719f, entries=150, sequenceid=116, filesize=30.2 K 2024-11-11T22:17:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/4db1a001b5cc4824be6a73de4e925e4f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/4db1a001b5cc4824be6a73de4e925e4f 2024-11-11T22:17:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,725 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/4db1a001b5cc4824be6a73de4e925e4f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T22:17:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/f0e9470c8bf140fa8abfe2fce062e844 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/f0e9470c8bf140fa8abfe2fce062e844 2024-11-11T22:17:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,748 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/f0e9470c8bf140fa8abfe2fce062e844, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T22:17:08,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,750 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 852ms, sequenceid=116, compaction requested=false 2024-11-11T22:17:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T22:17:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T22:17:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T22:17:08,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4060 sec 2024-11-11T22:17:08,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.4170 sec 2024-11-11T22:17:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:09,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:17:09,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:09,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:09,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:09,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:09,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:09,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:09,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116d787aa6d41244ab9eb1640aeafb6195_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363429197/Put/seqid=0 2024-11-11T22:17:09,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363489264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363489264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363489265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363489266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363489277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741968_1144 (size=12304) 2024-11-11T22:17:09,333 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:09,347 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116d787aa6d41244ab9eb1640aeafb6195_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d787aa6d41244ab9eb1640aeafb6195_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:09,356 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/da712457c9a7443b87dfde8b74227531, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:09,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/da712457c9a7443b87dfde8b74227531 is 175, key is test_row_0/A:col10/1731363429197/Put/seqid=0 2024-11-11T22:17:09,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741969_1145 (size=31105) 2024-11-11T22:17:09,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363489380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363489390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363489389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363489391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363489392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T22:17:09,520 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:09,520 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T22:17:09,523 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:09,524 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:09,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:17:09,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363489599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363489599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363489600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363489610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363489610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:17:09,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:09,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:09,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:09,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/da712457c9a7443b87dfde8b74227531 2024-11-11T22:17:09,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/acc5943f2900451ca492c920b32141fa is 50, key is test_row_0/B:col10/1731363429197/Put/seqid=0 2024-11-11T22:17:09,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741970_1146 (size=12151) 2024-11-11T22:17:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:17:09,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:09,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:09,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/acc5943f2900451ca492c920b32141fa 2024-11-11T22:17:09,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/2a9e89384530441a839b33cd57db23b3 is 50, key is test_row_0/C:col10/1731363429197/Put/seqid=0 2024-11-11T22:17:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363489910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363489910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363489910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741971_1147 (size=12151) 2024-11-11T22:17:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363489929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363489929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:09,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:09,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:10,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:10,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:17:10,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:10,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/2a9e89384530441a839b33cd57db23b3 2024-11-11T22:17:10,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/da712457c9a7443b87dfde8b74227531 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/da712457c9a7443b87dfde8b74227531 2024-11-11T22:17:10,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/da712457c9a7443b87dfde8b74227531, entries=150, sequenceid=133, filesize=30.4 K 2024-11-11T22:17:10,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/acc5943f2900451ca492c920b32141fa as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/acc5943f2900451ca492c920b32141fa 2024-11-11T22:17:10,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/acc5943f2900451ca492c920b32141fa, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T22:17:10,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/2a9e89384530441a839b33cd57db23b3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2a9e89384530441a839b33cd57db23b3 2024-11-11T22:17:10,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2a9e89384530441a839b33cd57db23b3, entries=150, sequenceid=133, filesize=11.9 K 2024-11-11T22:17:10,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1154ms, sequenceid=133, compaction requested=true 2024-11-11T22:17:10,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:10,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:10,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:10,376 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:10,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:10,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:10,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:10,376 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:10,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:10,377 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:10,378 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93118 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:10,378 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/B is initiating minor compaction (all files) 2024-11-11T22:17:10,378 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/A is initiating minor compaction (all files) 2024-11-11T22:17:10,378 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/B in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,378 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/A in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,378 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/c7617e2c90a8475cafe87996f6ff944f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/4db1a001b5cc4824be6a73de4e925e4f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/acc5943f2900451ca492c920b32141fa] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=35.4 K 2024-11-11T22:17:10,378 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0723b8fb54374f7ca9533c7baae089e3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2ee0416cb224f07a8d90761053f719f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/da712457c9a7443b87dfde8b74227531] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=90.9 K 2024-11-11T22:17:10,378 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,378 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0723b8fb54374f7ca9533c7baae089e3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2ee0416cb224f07a8d90761053f719f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/da712457c9a7443b87dfde8b74227531] 2024-11-11T22:17:10,379 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0723b8fb54374f7ca9533c7baae089e3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363426295 2024-11-11T22:17:10,379 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c7617e2c90a8475cafe87996f6ff944f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363426295 2024-11-11T22:17:10,379 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4db1a001b5cc4824be6a73de4e925e4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731363426694 2024-11-11T22:17:10,379 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2ee0416cb224f07a8d90761053f719f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731363426694 2024-11-11T22:17:10,380 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting acc5943f2900451ca492c920b32141fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731363429169 2024-11-11T22:17:10,380 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting da712457c9a7443b87dfde8b74227531, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731363429169 2024-11-11T22:17:10,407 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:10,408 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/00471ac475fa41e99dd02442e9478a06 is 50, key is test_row_0/B:col10/1731363429197/Put/seqid=0 2024-11-11T22:17:10,418 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:10,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:17:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:10,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:10,437 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111f93f20f34afb4867867e835f9fd69e3e_4c17d89df59d2cd5ce9ff2614859e6e4 store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:10,439 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111f93f20f34afb4867867e835f9fd69e3e_4c17d89df59d2cd5ce9ff2614859e6e4, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:10,440 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f93f20f34afb4867867e835f9fd69e3e_4c17d89df59d2cd5ce9ff2614859e6e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:10,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363490448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363490449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741972_1148 (size=12357) 2024-11-11T22:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363490450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363490451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363490453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:10,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:10,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116a549d14ae654695b74decd618670dcd_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363429260/Put/seqid=0 2024-11-11T22:17:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741973_1149 (size=4469) 2024-11-11T22:17:10,511 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#A#compaction#127 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:10,511 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/485eba5424bb4ce8aa8da522a620c169 is 175, key is test_row_0/A:col10/1731363429197/Put/seqid=0 2024-11-11T22:17:10,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741974_1150 (size=12304) 2024-11-11T22:17:10,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:10,529 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116a549d14ae654695b74decd618670dcd_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116a549d14ae654695b74decd618670dcd_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:10,533 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/11810bc6bd164157851b85466096b915, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:10,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/11810bc6bd164157851b85466096b915 is 175, key is test_row_0/A:col10/1731363429260/Put/seqid=0 2024-11-11T22:17:10,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741975_1151 (size=31311) 2024-11-11T22:17:10,555 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/485eba5424bb4ce8aa8da522a620c169 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/485eba5424bb4ce8aa8da522a620c169 2024-11-11T22:17:10,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363490560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363490560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,565 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/A of 4c17d89df59d2cd5ce9ff2614859e6e4 into 485eba5424bb4ce8aa8da522a620c169(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:10,565 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:10,565 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/A, priority=13, startTime=1731363430376; duration=0sec 2024-11-11T22:17:10,566 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:10,566 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:A 2024-11-11T22:17:10,566 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:10,567 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:10,567 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/C is initiating minor compaction (all files) 2024-11-11T22:17:10,567 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/C in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,568 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/072f63adb9be4efd8d0f826c2ff7796e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/f0e9470c8bf140fa8abfe2fce062e844, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2a9e89384530441a839b33cd57db23b3] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=35.4 K 2024-11-11T22:17:10,568 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 072f63adb9be4efd8d0f826c2ff7796e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731363426295 2024-11-11T22:17:10,569 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0e9470c8bf140fa8abfe2fce062e844, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731363426694 2024-11-11T22:17:10,569 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a9e89384530441a839b33cd57db23b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731363429169 2024-11-11T22:17:10,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363490570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363490570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363490579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741976_1152 (size=31105) 2024-11-11T22:17:10,595 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#C#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:10,596 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/c52a1602f01749459894b8ad18ec146c is 50, key is test_row_0/C:col10/1731363429197/Put/seqid=0 2024-11-11T22:17:10,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741977_1153 (size=12357) 2024-11-11T22:17:10,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:17:10,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363490769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363490769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363490780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363490780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:10,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363490790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:10,867 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/00471ac475fa41e99dd02442e9478a06 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/00471ac475fa41e99dd02442e9478a06 2024-11-11T22:17:10,878 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/B of 4c17d89df59d2cd5ce9ff2614859e6e4 into 00471ac475fa41e99dd02442e9478a06(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:10,878 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:10,878 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/B, priority=13, startTime=1731363430376; duration=0sec 2024-11-11T22:17:10,879 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:10,879 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:B 2024-11-11T22:17:10,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:10,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:10,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:10,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:10,997 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/11810bc6bd164157851b85466096b915 2024-11-11T22:17:11,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/55aadd76087b4394b5fc1530634b5a1c is 50, key is test_row_0/B:col10/1731363429260/Put/seqid=0 2024-11-11T22:17:11,042 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/c52a1602f01749459894b8ad18ec146c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c52a1602f01749459894b8ad18ec146c 2024-11-11T22:17:11,055 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/C of 4c17d89df59d2cd5ce9ff2614859e6e4 into c52a1602f01749459894b8ad18ec146c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:11,055 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:11,055 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/C, priority=13, startTime=1731363430376; duration=0sec 2024-11-11T22:17:11,055 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:11,055 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:C 2024-11-11T22:17:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741978_1154 (size=12151) 2024-11-11T22:17:11,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/55aadd76087b4394b5fc1530634b5a1c 2024-11-11T22:17:11,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363491080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363491081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363491101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363491104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363491102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:11,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:11,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7ba7a7a378b2468b9f582cb27860e3ae is 50, key is test_row_0/C:col10/1731363429260/Put/seqid=0 2024-11-11T22:17:11,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741979_1155 (size=12151) 2024-11-11T22:17:11,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:11,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:11,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7ba7a7a378b2468b9f582cb27860e3ae 2024-11-11T22:17:11,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/11810bc6bd164157851b85466096b915 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/11810bc6bd164157851b85466096b915 2024-11-11T22:17:11,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:11,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:11,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363491601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363491602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/11810bc6bd164157851b85466096b915, entries=150, sequenceid=152, filesize=30.4 K 2024-11-11T22:17:11,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/55aadd76087b4394b5fc1530634b5a1c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/55aadd76087b4394b5fc1530634b5a1c 2024-11-11T22:17:11,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363491610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363491616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:11,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363491620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:11,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:11,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/55aadd76087b4394b5fc1530634b5a1c, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T22:17:11,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7ba7a7a378b2468b9f582cb27860e3ae as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7ba7a7a378b2468b9f582cb27860e3ae 2024-11-11T22:17:11,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7ba7a7a378b2468b9f582cb27860e3ae, entries=150, sequenceid=152, filesize=11.9 K 2024-11-11T22:17:11,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1227ms, sequenceid=152, compaction requested=false 2024-11-11T22:17:11,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:17:11,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T22:17:11,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:11,757 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:17:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119e95093e9e54432292e45030122fb563_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363430450/Put/seqid=0 2024-11-11T22:17:11,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741980_1156 (size=12304) 2024-11-11T22:17:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:11,833 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119e95093e9e54432292e45030122fb563_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119e95093e9e54432292e45030122fb563_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:11,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/93084af1f75b4c009f73d7a7d3e4d736, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:11,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/93084af1f75b4c009f73d7a7d3e4d736 is 175, key is test_row_0/A:col10/1731363430450/Put/seqid=0 2024-11-11T22:17:11,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741981_1157 (size=31105) 2024-11-11T22:17:12,273 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/93084af1f75b4c009f73d7a7d3e4d736 2024-11-11T22:17:12,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/8617934588e348b9951768945af2f755 is 50, key is test_row_0/B:col10/1731363430450/Put/seqid=0 2024-11-11T22:17:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741982_1158 (size=12151) 2024-11-11T22:17:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:12,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363492666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363492678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363492691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363492692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363492695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,752 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/8617934588e348b9951768945af2f755 2024-11-11T22:17:12,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363492801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363492801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363492810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363492819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/6c1a049d86954a389704a02976665e9d is 50, key is test_row_0/C:col10/1731363430450/Put/seqid=0 2024-11-11T22:17:12,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363492820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:12,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741983_1159 (size=12151) 2024-11-11T22:17:13,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363493020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363493022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363493030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363493030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363493040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,055 INFO [master/8d82b89dd27d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T22:17:13,062 INFO [master/8d82b89dd27d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T22:17:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,288 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/6c1a049d86954a389704a02976665e9d 2024-11-11T22:17:13,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/93084af1f75b4c009f73d7a7d3e4d736 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/93084af1f75b4c009f73d7a7d3e4d736 2024-11-11T22:17:13,329 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/93084af1f75b4c009f73d7a7d3e4d736, entries=150, sequenceid=168, filesize=30.4 K 2024-11-11T22:17:13,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/8617934588e348b9951768945af2f755 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/8617934588e348b9951768945af2f755 2024-11-11T22:17:13,340 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/8617934588e348b9951768945af2f755, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T22:17:13,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/6c1a049d86954a389704a02976665e9d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6c1a049d86954a389704a02976665e9d 2024-11-11T22:17:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363493341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,352 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6c1a049d86954a389704a02976665e9d, entries=150, sequenceid=168, filesize=11.9 K 2024-11-11T22:17:13,357 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1601ms, sequenceid=168, compaction requested=true 2024-11-11T22:17:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T22:17:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T22:17:13,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T22:17:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:13,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363432649, max timestamp=1731363433361Num uniques 859; 2024-11-11T22:17:13,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:13,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363432649, max timestamp=1731363433361Num uniques 859; 2024-11-11T22:17:13,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:13,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:13,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T22:17:13,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:13,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:13,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8410 sec 2024-11-11T22:17:13,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:13,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:13,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:13,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 3.8580 sec 2024-11-11T22:17:13,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d39ed280eb0f45d6832395563882fe7d_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363433367/Put/seqid=0 2024-11-11T22:17:13,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741984_1160 (size=17284) 2024-11-11T22:17:13,435 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:13,444 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d39ed280eb0f45d6832395563882fe7d_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d39ed280eb0f45d6832395563882fe7d_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:13,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/66a41951d4ea46da99cf504f9460086b, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:13,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/66a41951d4ea46da99cf504f9460086b is 175, key is test_row_0/A:col10/1731363433367/Put/seqid=0 2024-11-11T22:17:13,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741985_1161 (size=48389) 2024-11-11T22:17:13,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731363433419, max timestamp=1731363433450Num uniques 859; 2024-11-11T22:17:13,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:13,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731363433419, max timestamp=1731363433450Num uniques 859; 2024-11-11T22:17:13,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:13,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9560, totalHeapSize=21976, min timestamp=1731363433419, max timestamp=1731363433450Num uniques 859; 2024-11-11T22:17:13,501 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39330, new segment data size=9560, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:13,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363493542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363493601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363493601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363493610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T22:17:13,680 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:13,681 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T22:17:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:17:13,686 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:13,687 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:13,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363493709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363493720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363493720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363493720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:17:13,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T22:17:13,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:13,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:13,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/66a41951d4ea46da99cf504f9460086b 2024-11-11T22:17:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:13,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363493864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/25de1f821660411cb24647d4d4a85136 is 50, key is test_row_0/B:col10/1731363433367/Put/seqid=0 2024-11-11T22:17:13,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363493921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741986_1162 (size=12151) 2024-11-11T22:17:13,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/25de1f821660411cb24647d4d4a85136 2024-11-11T22:17:13,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363493941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363493941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363493941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:13,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/c984d349049944d4906f09bd93bed67b is 50, key is test_row_0/C:col10/1731363433367/Put/seqid=0 2024-11-11T22:17:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:17:14,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741987_1163 (size=12151) 2024-11-11T22:17:14,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T22:17:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:14,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/c984d349049944d4906f09bd93bed67b 2024-11-11T22:17:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:14,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/66a41951d4ea46da99cf504f9460086b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/66a41951d4ea46da99cf504f9460086b 2024-11-11T22:17:14,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/66a41951d4ea46da99cf504f9460086b, entries=250, sequenceid=190, filesize=47.3 K 2024-11-11T22:17:14,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/25de1f821660411cb24647d4d4a85136 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/25de1f821660411cb24647d4d4a85136 2024-11-11T22:17:14,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/25de1f821660411cb24647d4d4a85136, entries=150, sequenceid=190, filesize=11.9 K 2024-11-11T22:17:14,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/c984d349049944d4906f09bd93bed67b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c984d349049944d4906f09bd93bed67b 2024-11-11T22:17:14,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c984d349049944d4906f09bd93bed67b, entries=150, sequenceid=190, filesize=11.9 K 2024-11-11T22:17:14,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=120.76 KB/123660 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 707ms, sequenceid=190, compaction requested=true 2024-11-11T22:17:14,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:14,076 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:14,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:14,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:14,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:14,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:14,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:14,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:17:14,077 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:14,080 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141910 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:14,080 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/A is initiating minor compaction (all files) 2024-11-11T22:17:14,080 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/A in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,080 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/485eba5424bb4ce8aa8da522a620c169, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/11810bc6bd164157851b85466096b915, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/93084af1f75b4c009f73d7a7d3e4d736, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/66a41951d4ea46da99cf504f9460086b] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=138.6 K 2024-11-11T22:17:14,080 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,080 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/485eba5424bb4ce8aa8da522a620c169, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/11810bc6bd164157851b85466096b915, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/93084af1f75b4c009f73d7a7d3e4d736, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/66a41951d4ea46da99cf504f9460086b] 2024-11-11T22:17:14,081 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:14,081 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/B is initiating minor compaction (all files) 2024-11-11T22:17:14,081 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/B in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,081 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/00471ac475fa41e99dd02442e9478a06, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/55aadd76087b4394b5fc1530634b5a1c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/8617934588e348b9951768945af2f755, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/25de1f821660411cb24647d4d4a85136] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=47.7 K 2024-11-11T22:17:14,088 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 485eba5424bb4ce8aa8da522a620c169, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731363429169 2024-11-11T22:17:14,088 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 00471ac475fa41e99dd02442e9478a06, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731363429169 2024-11-11T22:17:14,090 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 55aadd76087b4394b5fc1530634b5a1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731363429260 2024-11-11T22:17:14,090 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11810bc6bd164157851b85466096b915, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731363429260 2024-11-11T22:17:14,101 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8617934588e348b9951768945af2f755, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363430447 2024-11-11T22:17:14,101 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93084af1f75b4c009f73d7a7d3e4d736, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363430447 2024-11-11T22:17:14,105 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 25de1f821660411cb24647d4d4a85136, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731363432678 2024-11-11T22:17:14,105 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a41951d4ea46da99cf504f9460086b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731363432649 2024-11-11T22:17:14,151 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:14,152 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/b2461b4d1af94d36829bba0411a5fa72 is 50, key is test_row_0/B:col10/1731363433367/Put/seqid=0 2024-11-11T22:17:14,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T22:17:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,169 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-11T22:17:14,169 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:14,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:14,185 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411114428a11201414e0daa61a6a0b6aef2a0_4c17d89df59d2cd5ce9ff2614859e6e4 store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:14,188 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411114428a11201414e0daa61a6a0b6aef2a0_4c17d89df59d2cd5ce9ff2614859e6e4, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:14,188 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114428a11201414e0daa61a6a0b6aef2a0_4c17d89df59d2cd5ce9ff2614859e6e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:14,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741989_1165 (size=4469) 2024-11-11T22:17:14,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0afaab63b8a45a0bb78d82f8cd5af5b_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363433558/Put/seqid=0 2024-11-11T22:17:14,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741988_1164 (size=12493) 2024-11-11T22:17:14,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:14,266 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#A#compaction#139 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:14,267 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/9357dd684e6d48e1afb392eb642a2676 is 175, key is test_row_0/A:col10/1731363433367/Put/seqid=0 2024-11-11T22:17:14,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:17:14,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741991_1167 (size=31447) 2024-11-11T22:17:14,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741990_1166 (size=12304) 2024-11-11T22:17:14,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:14,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363494318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363494323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,352 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e0afaab63b8a45a0bb78d82f8cd5af5b_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0afaab63b8a45a0bb78d82f8cd5af5b_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:14,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363494339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363494341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,353 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/9357dd684e6d48e1afb392eb642a2676 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9357dd684e6d48e1afb392eb642a2676 2024-11-11T22:17:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b3fefb1301bf4ca1b064e95c94b9e0f6, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:14,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b3fefb1301bf4ca1b064e95c94b9e0f6 is 175, key is test_row_0/A:col10/1731363433558/Put/seqid=0 2024-11-11T22:17:14,367 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/A of 4c17d89df59d2cd5ce9ff2614859e6e4 into 9357dd684e6d48e1afb392eb642a2676(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:14,367 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:14,367 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/A, priority=12, startTime=1731363434076; duration=0sec 2024-11-11T22:17:14,368 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:14,368 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:A 2024-11-11T22:17:14,368 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:14,370 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:14,370 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/C is initiating minor compaction (all files) 2024-11-11T22:17:14,370 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/C in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,370 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c52a1602f01749459894b8ad18ec146c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7ba7a7a378b2468b9f582cb27860e3ae, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6c1a049d86954a389704a02976665e9d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c984d349049944d4906f09bd93bed67b] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=47.7 K 2024-11-11T22:17:14,371 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c52a1602f01749459894b8ad18ec146c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731363429169 2024-11-11T22:17:14,372 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ba7a7a378b2468b9f582cb27860e3ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731363429260 2024-11-11T22:17:14,372 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c1a049d86954a389704a02976665e9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731363430447 2024-11-11T22:17:14,373 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c984d349049944d4906f09bd93bed67b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731363432678 2024-11-11T22:17:14,387 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:14,388 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/9de2e1b05b434912884a8f9345cbbc5f is 50, key is test_row_0/C:col10/1731363433367/Put/seqid=0 2024-11-11T22:17:14,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741992_1168 (size=31105) 2024-11-11T22:17:14,393 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b3fefb1301bf4ca1b064e95c94b9e0f6 2024-11-11T22:17:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/cdb8c875872646dbaf0dedc78d74f170 is 50, key is test_row_0/B:col10/1731363433558/Put/seqid=0 2024-11-11T22:17:14,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741993_1169 (size=12493) 2024-11-11T22:17:14,432 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/9de2e1b05b434912884a8f9345cbbc5f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9de2e1b05b434912884a8f9345cbbc5f 2024-11-11T22:17:14,454 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/C of 4c17d89df59d2cd5ce9ff2614859e6e4 into 9de2e1b05b434912884a8f9345cbbc5f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:14,454 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:14,454 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/C, priority=12, startTime=1731363434077; duration=0sec 2024-11-11T22:17:14,454 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:14,454 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:C 2024-11-11T22:17:14,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741994_1170 (size=12151) 2024-11-11T22:17:14,456 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/cdb8c875872646dbaf0dedc78d74f170 2024-11-11T22:17:14,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363494452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363494452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363494461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363494461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7e4c7a04d0fb42e18bda0c6553e85fb1 is 50, key is test_row_0/C:col10/1731363433558/Put/seqid=0 2024-11-11T22:17:14,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741995_1171 (size=12151) 2024-11-11T22:17:14,606 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7e4c7a04d0fb42e18bda0c6553e85fb1 2024-11-11T22:17:14,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b3fefb1301bf4ca1b064e95c94b9e0f6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b3fefb1301bf4ca1b064e95c94b9e0f6 2024-11-11T22:17:14,661 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b3fefb1301bf4ca1b064e95c94b9e0f6, entries=150, sequenceid=224, filesize=30.4 K 2024-11-11T22:17:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/cdb8c875872646dbaf0dedc78d74f170 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/cdb8c875872646dbaf0dedc78d74f170 2024-11-11T22:17:14,683 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/cdb8c875872646dbaf0dedc78d74f170, entries=150, sequenceid=224, filesize=11.9 K 2024-11-11T22:17:14,686 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/b2461b4d1af94d36829bba0411a5fa72 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b2461b4d1af94d36829bba0411a5fa72 2024-11-11T22:17:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7e4c7a04d0fb42e18bda0c6553e85fb1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7e4c7a04d0fb42e18bda0c6553e85fb1 2024-11-11T22:17:14,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363494670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363494671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,704 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/B of 4c17d89df59d2cd5ce9ff2614859e6e4 into b2461b4d1af94d36829bba0411a5fa72(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:14,704 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:14,704 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/B, priority=12, startTime=1731363434077; duration=0sec 2024-11-11T22:17:14,705 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:14,705 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:B 2024-11-11T22:17:14,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363494680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363494680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:14,712 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7e4c7a04d0fb42e18bda0c6553e85fb1, entries=150, sequenceid=224, filesize=11.9 K 2024-11-11T22:17:14,716 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 547ms, sequenceid=224, compaction requested=false 2024-11-11T22:17:14,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:14,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T22:17:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T22:17:14,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T22:17:14,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-11T22:17:14,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.0430 sec 2024-11-11T22:17:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T22:17:14,822 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:14,824 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T22:17:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:17:14,828 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:14,829 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:14,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:14,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:17:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:14,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111118ae9d613934ef6a7668e1153ec5adc_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363434885/Put/seqid=0 2024-11-11T22:17:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:17:14,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741996_1172 (size=14794) 2024-11-11T22:17:14,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:14,983 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:14,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:14,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:14,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:14,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111118ae9d613934ef6a7668e1153ec5adc_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111118ae9d613934ef6a7668e1153ec5adc_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:15,032 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/c4a18dfa63114e5c8a18271ac177a6fe, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:15,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/c4a18dfa63114e5c8a18271ac177a6fe is 175, key is test_row_0/A:col10/1731363434885/Put/seqid=0 2024-11-11T22:17:15,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363495023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363495040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363495042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363495043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363495045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741997_1173 (size=39749) 2024-11-11T22:17:15,059 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/c4a18dfa63114e5c8a18271ac177a6fe 2024-11-11T22:17:15,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/0c657d4188424d1b810bee29ebe01417 is 50, key is test_row_0/B:col10/1731363434885/Put/seqid=0 2024-11-11T22:17:15,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741998_1174 (size=12151) 2024-11-11T22:17:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:17:15,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363495150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363495159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363495159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363495160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363495161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363495370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363495370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363495371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363495379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363495380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:17:15,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:15,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/0c657d4188424d1b810bee29ebe01417 2024-11-11T22:17:15,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/21bc2ca80bf14821b4a7ef08ef8b8381 is 50, key is test_row_0/C:col10/1731363434885/Put/seqid=0 2024-11-11T22:17:15,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741999_1175 (size=12151) 2024-11-11T22:17:15,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/21bc2ca80bf14821b4a7ef08ef8b8381 2024-11-11T22:17:15,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/c4a18dfa63114e5c8a18271ac177a6fe as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/c4a18dfa63114e5c8a18271ac177a6fe 2024-11-11T22:17:15,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/c4a18dfa63114e5c8a18271ac177a6fe, entries=200, sequenceid=240, filesize=38.8 K 2024-11-11T22:17:15,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/0c657d4188424d1b810bee29ebe01417 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0c657d4188424d1b810bee29ebe01417 2024-11-11T22:17:15,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0c657d4188424d1b810bee29ebe01417, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T22:17:15,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/21bc2ca80bf14821b4a7ef08ef8b8381 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/21bc2ca80bf14821b4a7ef08ef8b8381 2024-11-11T22:17:15,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/21bc2ca80bf14821b4a7ef08ef8b8381, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T22:17:15,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 777ms, sequenceid=240, compaction requested=true 2024-11-11T22:17:15,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:15,665 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:15,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:15,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:15,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:15,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:15,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:15,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:17:15,666 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:15,668 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:15,668 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:15,668 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/A is initiating minor compaction (all files) 2024-11-11T22:17:15,668 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/A in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,668 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9357dd684e6d48e1afb392eb642a2676, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b3fefb1301bf4ca1b064e95c94b9e0f6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/c4a18dfa63114e5c8a18271ac177a6fe] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=99.9 K 2024-11-11T22:17:15,668 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,668 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9357dd684e6d48e1afb392eb642a2676, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b3fefb1301bf4ca1b064e95c94b9e0f6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/c4a18dfa63114e5c8a18271ac177a6fe] 2024-11-11T22:17:15,668 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/B is initiating minor compaction (all files) 2024-11-11T22:17:15,669 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/B in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,669 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b2461b4d1af94d36829bba0411a5fa72, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/cdb8c875872646dbaf0dedc78d74f170, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0c657d4188424d1b810bee29ebe01417] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=35.9 K 2024-11-11T22:17:15,669 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9357dd684e6d48e1afb392eb642a2676, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731363432678 2024-11-11T22:17:15,670 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b2461b4d1af94d36829bba0411a5fa72, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731363432678 2024-11-11T22:17:15,670 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3fefb1301bf4ca1b064e95c94b9e0f6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731363433554 2024-11-11T22:17:15,670 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting cdb8c875872646dbaf0dedc78d74f170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731363433554 2024-11-11T22:17:15,671 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c657d4188424d1b810bee29ebe01417, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731363434332 2024-11-11T22:17:15,671 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a18dfa63114e5c8a18271ac177a6fe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731363434289 2024-11-11T22:17:15,699 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:15,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363435004, max timestamp=1731363435694Num uniques 859; 2024-11-11T22:17:15,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:15,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363435004, max timestamp=1731363435697Num uniques 859; 2024-11-11T22:17:15,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363435004, max timestamp=1731363435697Num uniques 859; 2024-11-11T22:17:15,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:15,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:15,705 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411117fb89a6c54644c32a76f336556719734_4c17d89df59d2cd5ce9ff2614859e6e4 store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:15,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-11T22:17:15,707 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:15,707 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/b4b97c0621374c34a529c48e83f51dde is 50, key is test_row_0/B:col10/1731363434885/Put/seqid=0 2024-11-11T22:17:15,708 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411117fb89a6c54644c32a76f336556719734_4c17d89df59d2cd5ce9ff2614859e6e4, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:15,708 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117fb89a6c54644c32a76f336556719734_4c17d89df59d2cd5ce9ff2614859e6e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:15,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:15,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:15,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:15,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:15,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:15,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:15,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ead602345795415faf174a8d7fd47d92_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363435700/Put/seqid=0 2024-11-11T22:17:15,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:15,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:15,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363435738, max timestamp=1731363435771Num uniques 859; 2024-11-11T22:17:15,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:15,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363435738, max timestamp=1731363435771Num uniques 859; 2024-11-11T22:17:15,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:15,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363435738, max timestamp=1731363435771Num uniques 859; 2024-11-11T22:17:15,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:15,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742001_1177 (size=12595) 2024-11-11T22:17:15,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742002_1178 (size=12454) 2024-11-11T22:17:15,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742000_1176 (size=4469) 2024-11-11T22:17:15,800 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#A#compaction#147 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:15,800 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/ba8eb70291bf44c487e7c42602402a51 is 175, key is test_row_0/A:col10/1731363434885/Put/seqid=0 2024-11-11T22:17:15,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742003_1179 (size=31549) 2024-11-11T22:17:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363495812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363495820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363495823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363495834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363495837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:15,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:15,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363495939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:17:15,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363495950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363495950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363495950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363495951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:16,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:16,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363496159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363496170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363496179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363496179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363496179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,192 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/b4b97c0621374c34a529c48e83f51dde as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b4b97c0621374c34a529c48e83f51dde 2024-11-11T22:17:16,197 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:16,202 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/B of 4c17d89df59d2cd5ce9ff2614859e6e4 into b4b97c0621374c34a529c48e83f51dde(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:16,202 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:16,202 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/B, priority=13, startTime=1731363435666; duration=0sec 2024-11-11T22:17:16,202 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:16,202 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:B 2024-11-11T22:17:16,202 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:16,205 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:16,205 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/C is initiating minor compaction (all files) 2024-11-11T22:17:16,205 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/C in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,205 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9de2e1b05b434912884a8f9345cbbc5f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7e4c7a04d0fb42e18bda0c6553e85fb1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/21bc2ca80bf14821b4a7ef08ef8b8381] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=35.9 K 2024-11-11T22:17:16,205 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ead602345795415faf174a8d7fd47d92_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ead602345795415faf174a8d7fd47d92_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:16,206 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de2e1b05b434912884a8f9345cbbc5f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731363432678 2024-11-11T22:17:16,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2d7b6a0b7f845509f278ac0a1d03abc, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:16,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2d7b6a0b7f845509f278ac0a1d03abc is 175, key is test_row_0/A:col10/1731363435700/Put/seqid=0 2024-11-11T22:17:16,208 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4c7a04d0fb42e18bda0c6553e85fb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731363433554 2024-11-11T22:17:16,209 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bc2ca80bf14821b4a7ef08ef8b8381, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731363434332 2024-11-11T22:17:16,231 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/ba8eb70291bf44c487e7c42602402a51 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/ba8eb70291bf44c487e7c42602402a51 2024-11-11T22:17:16,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,239 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/A of 4c17d89df59d2cd5ce9ff2614859e6e4 into ba8eb70291bf44c487e7c42602402a51(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:16,239 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:16,239 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/A, priority=13, startTime=1731363435665; duration=0sec 2024-11-11T22:17:16,239 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:16,239 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:A 2024-11-11T22:17:16,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742004_1180 (size=31255) 2024-11-11T22:17:16,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2d7b6a0b7f845509f278ac0a1d03abc 2024-11-11T22:17:16,251 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:16,252 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/9a670d6263c647b483635cfabbe8d864 is 50, key is test_row_0/C:col10/1731363434885/Put/seqid=0 2024-11-11T22:17:16,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/659378184bce4ea3aa21b300cfaa9e67 is 50, key is test_row_0/B:col10/1731363435700/Put/seqid=0 2024-11-11T22:17:16,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742006_1182 (size=12301) 2024-11-11T22:17:16,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742005_1181 (size=12595) 2024-11-11T22:17:16,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363496479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363496479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363496490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363496499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363496499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:16,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:16,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:16,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/659378184bce4ea3aa21b300cfaa9e67 2024-11-11T22:17:16,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/524dd7afe61c464faade92f276082045 is 50, key is test_row_0/C:col10/1731363435700/Put/seqid=0 2024-11-11T22:17:16,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742007_1183 (size=12301) 2024-11-11T22:17:16,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/524dd7afe61c464faade92f276082045 2024-11-11T22:17:16,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2d7b6a0b7f845509f278ac0a1d03abc as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2d7b6a0b7f845509f278ac0a1d03abc 2024-11-11T22:17:16,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2d7b6a0b7f845509f278ac0a1d03abc, entries=150, sequenceid=265, filesize=30.5 K 2024-11-11T22:17:16,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/659378184bce4ea3aa21b300cfaa9e67 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/659378184bce4ea3aa21b300cfaa9e67 2024-11-11T22:17:16,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/659378184bce4ea3aa21b300cfaa9e67, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T22:17:16,754 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T22:17:16,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/524dd7afe61c464faade92f276082045 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/524dd7afe61c464faade92f276082045 2024-11-11T22:17:16,755 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/9a670d6263c647b483635cfabbe8d864 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9a670d6263c647b483635cfabbe8d864 2024-11-11T22:17:16,760 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/C of 4c17d89df59d2cd5ce9ff2614859e6e4 into 9a670d6263c647b483635cfabbe8d864(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:16,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/524dd7afe61c464faade92f276082045, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T22:17:16,760 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:16,760 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/C, priority=13, startTime=1731363435666; duration=0sec 2024-11-11T22:17:16,760 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:16,760 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:C 2024-11-11T22:17:16,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1055ms, sequenceid=265, compaction requested=false 2024-11-11T22:17:16,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:16,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T22:17:16,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:16,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T22:17:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:16,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119959f52467294fabb5b77749940d772d_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363435834/Put/seqid=0 2024-11-11T22:17:16,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742008_1184 (size=12454) 2024-11-11T22:17:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:17:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:16,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:17,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363497010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363497010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363497010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363497011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363497011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363497119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363497119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:17,275 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119959f52467294fabb5b77749940d772d_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119959f52467294fabb5b77749940d772d_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:17,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d8943ada5d854ad0a701d93c1c5b7d84, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:17,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d8943ada5d854ad0a701d93c1c5b7d84 is 175, key is test_row_0/A:col10/1731363435834/Put/seqid=0 2024-11-11T22:17:17,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742009_1185 (size=31255) 2024-11-11T22:17:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363497329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363497329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363497649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363497649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:17,680 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d8943ada5d854ad0a701d93c1c5b7d84 2024-11-11T22:17:17,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/62e95ab778c841e0a87282bd4f9b3713 is 50, key is test_row_0/B:col10/1731363435834/Put/seqid=0 2024-11-11T22:17:17,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742010_1186 (size=12301) 2024-11-11T22:17:18,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:18,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363498019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:18,023 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 4c17d89df59d2cd5ce9ff2614859e6e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363498030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363498030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:18,032 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_1' in 4c17d89df59d2cd5ce9ff2614859e6e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:18,034 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 4c17d89df59d2cd5ce9ff2614859e6e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:18,099 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/62e95ab778c841e0a87282bd4f9b3713 2024-11-11T22:17:18,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5742a85da0534d37aa2688550f244dc6 is 50, key is test_row_0/C:col10/1731363435834/Put/seqid=0 2024-11-11T22:17:18,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742011_1187 (size=12301) 2024-11-11T22:17:18,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363498159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363498160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:18,521 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5742a85da0534d37aa2688550f244dc6 2024-11-11T22:17:18,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d8943ada5d854ad0a701d93c1c5b7d84 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d8943ada5d854ad0a701d93c1c5b7d84 2024-11-11T22:17:18,530 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d8943ada5d854ad0a701d93c1c5b7d84, entries=150, sequenceid=304, filesize=30.5 K 2024-11-11T22:17:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/62e95ab778c841e0a87282bd4f9b3713 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/62e95ab778c841e0a87282bd4f9b3713 2024-11-11T22:17:18,535 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/62e95ab778c841e0a87282bd4f9b3713, entries=150, sequenceid=304, filesize=12.0 K 2024-11-11T22:17:18,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5742a85da0534d37aa2688550f244dc6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5742a85da0534d37aa2688550f244dc6 2024-11-11T22:17:18,541 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5742a85da0534d37aa2688550f244dc6, entries=150, sequenceid=304, filesize=12.0 K 2024-11-11T22:17:18,542 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1684ms, sequenceid=304, compaction requested=true 2024-11-11T22:17:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T22:17:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T22:17:18,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T22:17:18,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7160 sec 2024-11-11T22:17:18,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.7230 sec 2024-11-11T22:17:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T22:17:18,990 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:18,990 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T22:17:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:17:18,993 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:18,994 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:18,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:17:19,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T22:17:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:19,146 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T22:17:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:19,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:19,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:19,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:19,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:19,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116d60d8d08be047ce896bc308a61a7de0_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363437009/Put/seqid=0 2024-11-11T22:17:19,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742012_1188 (size=12454) 2024-11-11T22:17:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:19,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363499219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:19,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363439206, max timestamp=1731363439219Num uniques 859; 2024-11-11T22:17:19,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:19,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363439206, max timestamp=1731363439219Num uniques 859; 2024-11-11T22:17:19,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363439206, max timestamp=1731363439219Num uniques 859; 2024-11-11T22:17:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:19,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:19,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363499299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:17:19,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363499329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:19,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:19,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363499409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:19,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363499539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:19,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:19,565 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116d60d8d08be047ce896bc308a61a7de0_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d60d8d08be047ce896bc308a61a7de0_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2292ba9ab7646ddabd90c11cf205312, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:19,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2292ba9ab7646ddabd90c11cf205312 is 175, key is test_row_0/A:col10/1731363437009/Put/seqid=0 2024-11-11T22:17:19,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742013_1189 (size=31255) 2024-11-11T22:17:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:17:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363499619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:19,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:19,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363499849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:19,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363499930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:19,977 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2292ba9ab7646ddabd90c11cf205312 2024-11-11T22:17:19,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/befc9d3192e345939895f56e6c20b256 is 50, key is test_row_0/B:col10/1731363437009/Put/seqid=0 2024-11-11T22:17:19,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742014_1190 (size=12301) 2024-11-11T22:17:20,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363500029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:20,035 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 4c17d89df59d2cd5ce9ff2614859e6e4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:20,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363500040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:20,049 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 4c17d89df59d2cd5ce9ff2614859e6e4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:20,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363500049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:20,054 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_1' in 4c17d89df59d2cd5ce9ff2614859e6e4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:17:20,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363500369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:20,390 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/befc9d3192e345939895f56e6c20b256 2024-11-11T22:17:20,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/a2183bf499a44eb4bd662eb9248ff91c is 50, key is test_row_0/C:col10/1731363437009/Put/seqid=0 2024-11-11T22:17:20,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742015_1191 (size=12301) 2024-11-11T22:17:20,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363500439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:20,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:20,801 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/a2183bf499a44eb4bd662eb9248ff91c 2024-11-11T22:17:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/b2292ba9ab7646ddabd90c11cf205312 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2292ba9ab7646ddabd90c11cf205312 2024-11-11T22:17:20,810 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2292ba9ab7646ddabd90c11cf205312, entries=150, sequenceid=315, filesize=30.5 K 2024-11-11T22:17:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/befc9d3192e345939895f56e6c20b256 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/befc9d3192e345939895f56e6c20b256 2024-11-11T22:17:20,816 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/befc9d3192e345939895f56e6c20b256, entries=150, sequenceid=315, filesize=12.0 K 2024-11-11T22:17:20,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/a2183bf499a44eb4bd662eb9248ff91c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/a2183bf499a44eb4bd662eb9248ff91c 2024-11-11T22:17:20,822 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/a2183bf499a44eb4bd662eb9248ff91c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-11T22:17:20,823 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1677ms, sequenceid=315, compaction requested=true 2024-11-11T22:17:20,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:20,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:20,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T22:17:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T22:17:20,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T22:17:20,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-11-11T22:17:20,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8370 sec 2024-11-11T22:17:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T22:17:21,140 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:21,140 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-11T22:17:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:17:21,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:21,145 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:21,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:17:21,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-11T22:17:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:21,298 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T22:17:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:21,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114c776981f4ef402a82dbac2398913153_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363439297/Put/seqid=0 2024-11-11T22:17:21,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742016_1192 (size=12454) 2024-11-11T22:17:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363501430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:17:21,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363501459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:21,460 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_0' in 4c17d89df59d2cd5ce9ff2614859e6e4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363501539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:21,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:21,716 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114c776981f4ef402a82dbac2398913153_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c776981f4ef402a82dbac2398913153_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:21,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/f75cd587dba94cb0ae916503f30a549d, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:21,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/f75cd587dba94cb0ae916503f30a549d is 175, key is test_row_0/A:col10/1731363439297/Put/seqid=0 2024-11-11T22:17:21,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742017_1193 (size=31255) 2024-11-11T22:17:21,725 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/f75cd587dba94cb0ae916503f30a549d 2024-11-11T22:17:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/599198ec7253479aa57de1ab64c3b218 is 50, key is test_row_0/B:col10/1731363439297/Put/seqid=0 2024-11-11T22:17:21,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742018_1194 (size=12301) 2024-11-11T22:17:21,737 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/599198ec7253479aa57de1ab64c3b218 2024-11-11T22:17:21,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/3772cc22c99045b1baa0591cedc7de8b is 50, key is test_row_0/C:col10/1731363439297/Put/seqid=0 2024-11-11T22:17:21,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363501749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:21,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742019_1195 (size=12301) 2024-11-11T22:17:21,761 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/3772cc22c99045b1baa0591cedc7de8b 2024-11-11T22:17:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/f75cd587dba94cb0ae916503f30a549d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f75cd587dba94cb0ae916503f30a549d 2024-11-11T22:17:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:17:21,771 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f75cd587dba94cb0ae916503f30a549d, entries=150, sequenceid=350, filesize=30.5 K 2024-11-11T22:17:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/599198ec7253479aa57de1ab64c3b218 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/599198ec7253479aa57de1ab64c3b218 2024-11-11T22:17:21,779 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/599198ec7253479aa57de1ab64c3b218, entries=150, sequenceid=350, filesize=12.0 K 2024-11-11T22:17:21,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/3772cc22c99045b1baa0591cedc7de8b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/3772cc22c99045b1baa0591cedc7de8b 2024-11-11T22:17:21,787 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/3772cc22c99045b1baa0591cedc7de8b, entries=150, sequenceid=350, filesize=12.0 K 2024-11-11T22:17:21,790 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 492ms, sequenceid=350, compaction requested=true 2024-11-11T22:17:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:21,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-11T22:17:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-11T22:17:21,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T22:17:21,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 647 msec 2024-11-11T22:17:21,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 655 msec 2024-11-11T22:17:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:22,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:17:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:22,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b9088347ee9a41af831e33eab4d90710_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363441428/Put/seqid=0 2024-11-11T22:17:22,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742020_1196 (size=14994) 2024-11-11T22:17:22,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363502183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T22:17:22,280 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:22,281 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-11T22:17:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:17:22,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:22,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:22,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:22,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363502299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:22,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:17:22,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T22:17:22,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:22,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:22,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:22,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,518 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363502509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:22,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b9088347ee9a41af831e33eab4d90710_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b9088347ee9a41af831e33eab4d90710_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:22,533 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/9fc04daf7f1c42ec84c404c7b2ff427c, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:22,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/9fc04daf7f1c42ec84c404c7b2ff427c is 175, key is test_row_0/A:col10/1731363441428/Put/seqid=0 2024-11-11T22:17:22,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742021_1197 (size=39949) 2024-11-11T22:17:22,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/9fc04daf7f1c42ec84c404c7b2ff427c 2024-11-11T22:17:22,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T22:17:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:22,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:17:22,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/26ab948200c144688015434e94c323a2 is 50, key is test_row_0/B:col10/1731363441428/Put/seqid=0 2024-11-11T22:17:22,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742022_1198 (size=12301) 2024-11-11T22:17:22,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T22:17:22,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:22,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:22,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:22,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363502830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:17:22,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T22:17:22,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:22,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:22,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:23,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/26ab948200c144688015434e94c323a2 2024-11-11T22:17:23,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/aea4952d3e53466291ba267594e134c6 is 50, key is test_row_0/C:col10/1731363441428/Put/seqid=0 2024-11-11T22:17:23,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742023_1199 (size=12301) 2024-11-11T22:17:23,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/aea4952d3e53466291ba267594e134c6 2024-11-11T22:17:23,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T22:17:23,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:23,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:23,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:23,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:23,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:23,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/9fc04daf7f1c42ec84c404c7b2ff427c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9fc04daf7f1c42ec84c404c7b2ff427c 2024-11-11T22:17:23,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9fc04daf7f1c42ec84c404c7b2ff427c, entries=200, sequenceid=362, filesize=39.0 K 2024-11-11T22:17:23,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/26ab948200c144688015434e94c323a2 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/26ab948200c144688015434e94c323a2 2024-11-11T22:17:23,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/26ab948200c144688015434e94c323a2, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T22:17:23,147 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T22:17:23,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/aea4952d3e53466291ba267594e134c6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/aea4952d3e53466291ba267594e134c6 2024-11-11T22:17:23,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/aea4952d3e53466291ba267594e134c6, entries=150, sequenceid=362, filesize=12.0 K 2024-11-11T22:17:23,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1094ms, sequenceid=362, compaction requested=true 2024-11-11T22:17:23,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:23,168 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T22:17:23,168 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T22:17:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:23,170 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74100 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T22:17:23,170 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 196518 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T22:17:23,170 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/A is initiating minor compaction (all files) 2024-11-11T22:17:23,170 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/B is initiating minor compaction (all files) 2024-11-11T22:17:23,170 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/A in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:23,170 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/ba8eb70291bf44c487e7c42602402a51, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2d7b6a0b7f845509f278ac0a1d03abc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d8943ada5d854ad0a701d93c1c5b7d84, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2292ba9ab7646ddabd90c11cf205312, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f75cd587dba94cb0ae916503f30a549d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9fc04daf7f1c42ec84c404c7b2ff427c] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=191.9 K 2024-11-11T22:17:23,170 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:23,171 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/ba8eb70291bf44c487e7c42602402a51, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2d7b6a0b7f845509f278ac0a1d03abc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d8943ada5d854ad0a701d93c1c5b7d84, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2292ba9ab7646ddabd90c11cf205312, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f75cd587dba94cb0ae916503f30a549d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9fc04daf7f1c42ec84c404c7b2ff427c] 2024-11-11T22:17:23,171 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/B in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:23,171 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba8eb70291bf44c487e7c42602402a51, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731363434332 2024-11-11T22:17:23,171 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b4b97c0621374c34a529c48e83f51dde, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/659378184bce4ea3aa21b300cfaa9e67, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/62e95ab778c841e0a87282bd4f9b3713, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/befc9d3192e345939895f56e6c20b256, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/599198ec7253479aa57de1ab64c3b218, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/26ab948200c144688015434e94c323a2] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=72.4 K 2024-11-11T22:17:23,172 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d7b6a0b7f845509f278ac0a1d03abc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731363435700 2024-11-11T22:17:23,172 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b97c0621374c34a529c48e83f51dde, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731363434332 2024-11-11T22:17:23,172 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8943ada5d854ad0a701d93c1c5b7d84, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363435820 2024-11-11T22:17:23,172 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2292ba9ab7646ddabd90c11cf205312, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731363436999 2024-11-11T22:17:23,173 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f75cd587dba94cb0ae916503f30a549d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731363439281 2024-11-11T22:17:23,173 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 659378184bce4ea3aa21b300cfaa9e67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731363435700 2024-11-11T22:17:23,173 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc04daf7f1c42ec84c404c7b2ff427c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731363441408 2024-11-11T22:17:23,173 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e95ab778c841e0a87282bd4f9b3713, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363435820 2024-11-11T22:17:23,177 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting befc9d3192e345939895f56e6c20b256, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731363436999 2024-11-11T22:17:23,178 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 599198ec7253479aa57de1ab64c3b218, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731363439281 2024-11-11T22:17:23,178 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ab948200c144688015434e94c323a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731363441411 2024-11-11T22:17:23,200 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#B#compaction#165 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:23,201 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/34ae61a1ffc94456a916d9c4ee9b2105 is 50, key is test_row_0/B:col10/1731363441428/Put/seqid=0 2024-11-11T22:17:23,201 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:23,211 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411116256941d7d284306b76a44a2b5c879f4_4c17d89df59d2cd5ce9ff2614859e6e4 store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:23,216 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411116256941d7d284306b76a44a2b5c879f4_4c17d89df59d2cd5ce9ff2614859e6e4, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:23,217 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116256941d7d284306b76a44a2b5c879f4_4c17d89df59d2cd5ce9ff2614859e6e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:23,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T22:17:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:23,233 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:17:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:23,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742025_1201 (size=4469) 2024-11-11T22:17:23,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742024_1200 (size=12949) 2024-11-11T22:17:23,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a6eee0c551ac461aa0ac221503aced25_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363442181/Put/seqid=0 2024-11-11T22:17:23,296 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/34ae61a1ffc94456a916d9c4ee9b2105 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/34ae61a1ffc94456a916d9c4ee9b2105 2024-11-11T22:17:23,309 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/B of 4c17d89df59d2cd5ce9ff2614859e6e4 into 34ae61a1ffc94456a916d9c4ee9b2105(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:23,309 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:23,309 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/B, priority=10, startTime=1731363443168; duration=0sec 2024-11-11T22:17:23,309 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:23,309 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:B 2024-11-11T22:17:23,309 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T22:17:23,311 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74100 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T22:17:23,311 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/C is initiating minor compaction (all files) 2024-11-11T22:17:23,312 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/C in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:23,312 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9a670d6263c647b483635cfabbe8d864, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/524dd7afe61c464faade92f276082045, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5742a85da0534d37aa2688550f244dc6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/a2183bf499a44eb4bd662eb9248ff91c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/3772cc22c99045b1baa0591cedc7de8b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/aea4952d3e53466291ba267594e134c6] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=72.4 K 2024-11-11T22:17:23,312 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a670d6263c647b483635cfabbe8d864, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731363434332 2024-11-11T22:17:23,313 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 524dd7afe61c464faade92f276082045, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731363435700 2024-11-11T22:17:23,313 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5742a85da0534d37aa2688550f244dc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363435820 2024-11-11T22:17:23,314 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a2183bf499a44eb4bd662eb9248ff91c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731363436999 2024-11-11T22:17:23,314 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3772cc22c99045b1baa0591cedc7de8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731363439281 2024-11-11T22:17:23,314 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting aea4952d3e53466291ba267594e134c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731363441411 2024-11-11T22:17:23,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742026_1202 (size=12454) 2024-11-11T22:17:23,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:23,338 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#C#compaction#168 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:23,338 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/467f4d50fb49484cbfe219850bbf5599 is 50, key is test_row_0/C:col10/1731363441428/Put/seqid=0 2024-11-11T22:17:23,342 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a6eee0c551ac461aa0ac221503aced25_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a6eee0c551ac461aa0ac221503aced25_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/3a58f01fdd9142e496400da5b41ad4e1, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:23,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/3a58f01fdd9142e496400da5b41ad4e1 is 175, key is test_row_0/A:col10/1731363442181/Put/seqid=0 2024-11-11T22:17:23,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:23,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742028_1204 (size=31255) 2024-11-11T22:17:23,400 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/3a58f01fdd9142e496400da5b41ad4e1 2024-11-11T22:17:23,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/14dd99255629481c9f2771fea5b932b6 is 50, key is test_row_0/B:col10/1731363442181/Put/seqid=0 2024-11-11T22:17:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:17:23,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742027_1203 (size=12949) 2024-11-11T22:17:23,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742029_1205 (size=12301) 2024-11-11T22:17:23,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363503508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:23,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363503511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:23,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363503633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363503636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:23,680 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#A#compaction#166 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:23,681 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/21565db96a5940b3a3c2394f26a4b575 is 175, key is test_row_0/A:col10/1731363441428/Put/seqid=0 2024-11-11T22:17:23,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742030_1206 (size=31903) 2024-11-11T22:17:23,844 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/467f4d50fb49484cbfe219850bbf5599 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/467f4d50fb49484cbfe219850bbf5599 2024-11-11T22:17:23,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363503841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:23,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363503850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:23,859 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/C of 4c17d89df59d2cd5ce9ff2614859e6e4 into 467f4d50fb49484cbfe219850bbf5599(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:23,859 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:23,859 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/C, priority=10, startTime=1731363443168; duration=0sec 2024-11-11T22:17:23,859 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:23,859 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:C 2024-11-11T22:17:23,863 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/14dd99255629481c9f2771fea5b932b6 2024-11-11T22:17:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5f5e16578b194070bb4ccb6463c2bd36 is 50, key is test_row_0/C:col10/1731363442181/Put/seqid=0 2024-11-11T22:17:23,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742031_1207 (size=12301) 2024-11-11T22:17:23,925 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5f5e16578b194070bb4ccb6463c2bd36 2024-11-11T22:17:23,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/3a58f01fdd9142e496400da5b41ad4e1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/3a58f01fdd9142e496400da5b41ad4e1 2024-11-11T22:17:23,939 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/3a58f01fdd9142e496400da5b41ad4e1, entries=150, sequenceid=382, filesize=30.5 K 2024-11-11T22:17:23,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/14dd99255629481c9f2771fea5b932b6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/14dd99255629481c9f2771fea5b932b6 2024-11-11T22:17:23,947 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/14dd99255629481c9f2771fea5b932b6, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T22:17:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5f5e16578b194070bb4ccb6463c2bd36 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f5e16578b194070bb4ccb6463c2bd36 2024-11-11T22:17:23,957 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f5e16578b194070bb4ccb6463c2bd36, entries=150, sequenceid=382, filesize=12.0 K 2024-11-11T22:17:23,959 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 727ms, sequenceid=382, compaction requested=false 2024-11-11T22:17:23,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:23,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:23,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-11T22:17:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-11T22:17:23,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T22:17:23,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6780 sec 2024-11-11T22:17:23,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.6890 sec 2024-11-11T22:17:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:24,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:17:24,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:24,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:24,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:24,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:24,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:24,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:24,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113444d95f3c8843c0a7fb6344210de9a5_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363443497/Put/seqid=0 2024-11-11T22:17:24,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742032_1208 (size=14994) 2024-11-11T22:17:24,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363504108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,128 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/21565db96a5940b3a3c2394f26a4b575 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/21565db96a5940b3a3c2394f26a4b575 2024-11-11T22:17:24,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363504118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363504119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,138 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/A of 4c17d89df59d2cd5ce9ff2614859e6e4 into 21565db96a5940b3a3c2394f26a4b575(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:24,138 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:24,138 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/A, priority=10, startTime=1731363443167; duration=0sec 2024-11-11T22:17:24,138 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:24,138 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:A 2024-11-11T22:17:24,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363504160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363504161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363504231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363504242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363504242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T22:17:24,430 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:24,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-11T22:17:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T22:17:24,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:24,435 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:24,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:24,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363504449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363504462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363504478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,512 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:24,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113444d95f3c8843c0a7fb6344210de9a5_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113444d95f3c8843c0a7fb6344210de9a5_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:24,520 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d592c8e597004e9f94c44bb0ee7acb81, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:24,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d592c8e597004e9f94c44bb0ee7acb81 is 175, key is test_row_0/A:col10/1731363443497/Put/seqid=0 2024-11-11T22:17:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T22:17:24,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742033_1209 (size=39949) 2024-11-11T22:17:24,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d592c8e597004e9f94c44bb0ee7acb81 2024-11-11T22:17:24,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/f5332decb60243638a44f1cdbf075c7a is 50, key is test_row_0/B:col10/1731363443497/Put/seqid=0 2024-11-11T22:17:24,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T22:17:24,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:24,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:24,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:24,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:24,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:24,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742034_1210 (size=12301) 2024-11-11T22:17:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41984 deadline: 1731363504679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42022 deadline: 1731363504679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T22:17:24,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:24,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:24,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T22:17:24,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363504759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363504779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363504789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:24,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T22:17:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:24,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/f5332decb60243638a44f1cdbf075c7a 2024-11-11T22:17:25,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/6fca93905e1f4181aa0cf4252c076454 is 50, key is test_row_0/C:col10/1731363443497/Put/seqid=0 2024-11-11T22:17:25,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742035_1211 (size=12301) 2024-11-11T22:17:25,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T22:17:25,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:25,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T22:17:25,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T22:17:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42034 deadline: 1731363505269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:25,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:25,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:25,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41978 deadline: 1731363505289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:25,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42024 deadline: 1731363505299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:17:25,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T22:17:25,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:25,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:25,363 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-11T22:17:25,363 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:25,363 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,363 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:25,366 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-11T22:17:25,366 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:25,366 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,367 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:25,367 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-11T22:17:25,367 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:25,367 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,368 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:25,369 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-11T22:17:25,369 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:25,369 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,369 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:25,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/6fca93905e1f4181aa0cf4252c076454 2024-11-11T22:17:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/d592c8e597004e9f94c44bb0ee7acb81 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d592c8e597004e9f94c44bb0ee7acb81 2024-11-11T22:17:25,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d592c8e597004e9f94c44bb0ee7acb81, entries=200, sequenceid=397, filesize=39.0 K 2024-11-11T22:17:25,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/f5332decb60243638a44f1cdbf075c7a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/f5332decb60243638a44f1cdbf075c7a 2024-11-11T22:17:25,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/f5332decb60243638a44f1cdbf075c7a, entries=150, sequenceid=397, filesize=12.0 K 2024-11-11T22:17:25,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/6fca93905e1f4181aa0cf4252c076454 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6fca93905e1f4181aa0cf4252c076454 2024-11-11T22:17:25,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6fca93905e1f4181aa0cf4252c076454, entries=150, sequenceid=397, filesize=12.0 K 2024-11-11T22:17:25,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1400ms, sequenceid=397, compaction requested=true 2024-11-11T22:17:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:25,455 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:25,455 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c17d89df59d2cd5ce9ff2614859e6e4:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:25,456 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:25,456 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/A is initiating minor compaction (all files) 2024-11-11T22:17:25,456 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:25,456 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/A in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,456 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/B is initiating minor compaction (all files) 2024-11-11T22:17:25,456 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/B in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,456 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/21565db96a5940b3a3c2394f26a4b575, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/3a58f01fdd9142e496400da5b41ad4e1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d592c8e597004e9f94c44bb0ee7acb81] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=100.7 K 2024-11-11T22:17:25,456 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,456 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/34ae61a1ffc94456a916d9c4ee9b2105, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/14dd99255629481c9f2771fea5b932b6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/f5332decb60243638a44f1cdbf075c7a] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=36.7 K 2024-11-11T22:17:25,456 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/21565db96a5940b3a3c2394f26a4b575, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/3a58f01fdd9142e496400da5b41ad4e1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d592c8e597004e9f94c44bb0ee7acb81] 2024-11-11T22:17:25,456 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21565db96a5940b3a3c2394f26a4b575, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731363441411 2024-11-11T22:17:25,456 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ae61a1ffc94456a916d9c4ee9b2105, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731363441411 2024-11-11T22:17:25,457 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a58f01fdd9142e496400da5b41ad4e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731363442166 2024-11-11T22:17:25,457 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 14dd99255629481c9f2771fea5b932b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731363442166 2024-11-11T22:17:25,457 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f5332decb60243638a44f1cdbf075c7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731363443456 2024-11-11T22:17:25,457 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d592c8e597004e9f94c44bb0ee7acb81, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731363443456 2024-11-11T22:17:25,468 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:25,470 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#B#compaction#174 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:25,471 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/43323b33bdf445ffb365b067b7412ee8 is 50, key is test_row_0/B:col10/1731363443497/Put/seqid=0 2024-11-11T22:17:25,474 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111741b5690d0c149ee893951fcb3516927_4c17d89df59d2cd5ce9ff2614859e6e4 store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:25,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742036_1212 (size=13051) 2024-11-11T22:17:25,498 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111741b5690d0c149ee893951fcb3516927_4c17d89df59d2cd5ce9ff2614859e6e4, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:25,498 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111741b5690d0c149ee893951fcb3516927_4c17d89df59d2cd5ce9ff2614859e6e4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:25,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T22:17:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,513 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:17:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:25,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742037_1213 (size=4469) 2024-11-11T22:17:25,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a8dd57e2493848d68152054a381984a1_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363444068/Put/seqid=0 2024-11-11T22:17:25,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742038_1214 (size=12454) 2024-11-11T22:17:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T22:17:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:25,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. as already flushing 2024-11-11T22:17:25,701 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-11T22:17:25,702 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-11T22:17:25,702 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:25,702 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,702 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:25,702 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,702 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,702 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:25,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:25,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:25,890 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/43323b33bdf445ffb365b067b7412ee8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/43323b33bdf445ffb365b067b7412ee8 2024-11-11T22:17:25,897 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/B of 4c17d89df59d2cd5ce9ff2614859e6e4 into 43323b33bdf445ffb365b067b7412ee8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:25,897 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:25,897 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/B, priority=13, startTime=1731363445455; duration=0sec 2024-11-11T22:17:25,897 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:25,897 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:B 2024-11-11T22:17:25,897 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:25,899 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:25,899 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 4c17d89df59d2cd5ce9ff2614859e6e4/C is initiating minor compaction (all files) 2024-11-11T22:17:25,899 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c17d89df59d2cd5ce9ff2614859e6e4/C in TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:25,899 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/467f4d50fb49484cbfe219850bbf5599, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f5e16578b194070bb4ccb6463c2bd36, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6fca93905e1f4181aa0cf4252c076454] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp, totalSize=36.7 K 2024-11-11T22:17:25,899 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 467f4d50fb49484cbfe219850bbf5599, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731363441411 2024-11-11T22:17:25,900 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5e16578b194070bb4ccb6463c2bd36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731363442166 2024-11-11T22:17:25,900 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fca93905e1f4181aa0cf4252c076454, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731363443456 2024-11-11T22:17:25,913 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#C#compaction#177 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:25,913 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/421dae5922c14976b844bfdeab02cf86 is 50, key is test_row_0/C:col10/1731363443497/Put/seqid=0 2024-11-11T22:17:25,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742039_1215 (size=13051) 2024-11-11T22:17:25,920 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c17d89df59d2cd5ce9ff2614859e6e4#A#compaction#175 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:25,920 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/0e0e262c929f45908247beae79d7e185 is 175, key is test_row_0/A:col10/1731363443497/Put/seqid=0 2024-11-11T22:17:25,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742040_1216 (size=32005) 2024-11-11T22:17:25,925 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/421dae5922c14976b844bfdeab02cf86 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/421dae5922c14976b844bfdeab02cf86 2024-11-11T22:17:25,928 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/0e0e262c929f45908247beae79d7e185 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0e0e262c929f45908247beae79d7e185 2024-11-11T22:17:25,929 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/C of 4c17d89df59d2cd5ce9ff2614859e6e4 into 421dae5922c14976b844bfdeab02cf86(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:25,929 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:25,929 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/C, priority=13, startTime=1731363445455; duration=0sec 2024-11-11T22:17:25,929 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:25,929 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:C 2024-11-11T22:17:25,933 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c17d89df59d2cd5ce9ff2614859e6e4/A of 4c17d89df59d2cd5ce9ff2614859e6e4 into 0e0e262c929f45908247beae79d7e185(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:25,933 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:25,933 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., storeName=4c17d89df59d2cd5ce9ff2614859e6e4/A, priority=13, startTime=1731363445455; duration=0sec 2024-11-11T22:17:25,933 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:25,933 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c17d89df59d2cd5ce9ff2614859e6e4:A 2024-11-11T22:17:25,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:25,944 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a8dd57e2493848d68152054a381984a1_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a8dd57e2493848d68152054a381984a1_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/e7892c11b5ed4c42b4d9b6e8b64fbd9b, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/e7892c11b5ed4c42b4d9b6e8b64fbd9b is 175, key is test_row_0/A:col10/1731363444068/Put/seqid=0 2024-11-11T22:17:25,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742041_1217 (size=31255) 2024-11-11T22:17:26,250 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:17:26,293 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-11T22:17:26,293 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:26,293 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:26,294 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:26,294 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:26,312 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-11T22:17:26,312 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:26,312 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:26,313 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:26,313 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:26,323 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-11T22:17:26,323 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:26,323 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:26,323 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:26,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:26,350 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/e7892c11b5ed4c42b4d9b6e8b64fbd9b 2024-11-11T22:17:26,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/6bb6af79fc874db9b532b7e28934e42a is 50, key is test_row_0/B:col10/1731363444068/Put/seqid=0 2024-11-11T22:17:26,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742042_1218 (size=12301) 2024-11-11T22:17:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T22:17:26,766 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/6bb6af79fc874db9b532b7e28934e42a 2024-11-11T22:17:26,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7b6d9968dde74d368b9e78df4fbff310 is 50, key is test_row_0/C:col10/1731363444068/Put/seqid=0 2024-11-11T22:17:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742043_1219 (size=12301) 2024-11-11T22:17:27,177 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7b6d9968dde74d368b9e78df4fbff310 2024-11-11T22:17:27,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/e7892c11b5ed4c42b4d9b6e8b64fbd9b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/e7892c11b5ed4c42b4d9b6e8b64fbd9b 2024-11-11T22:17:27,187 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/e7892c11b5ed4c42b4d9b6e8b64fbd9b, entries=150, sequenceid=417, filesize=30.5 K 2024-11-11T22:17:27,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/6bb6af79fc874db9b532b7e28934e42a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/6bb6af79fc874db9b532b7e28934e42a 2024-11-11T22:17:27,191 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/6bb6af79fc874db9b532b7e28934e42a, entries=150, sequenceid=417, filesize=12.0 K 2024-11-11T22:17:27,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/7b6d9968dde74d368b9e78df4fbff310 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7b6d9968dde74d368b9e78df4fbff310 2024-11-11T22:17:27,196 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7b6d9968dde74d368b9e78df4fbff310, entries=150, sequenceid=417, filesize=12.0 K 2024-11-11T22:17:27,197 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=33.54 KB/34350 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1684ms, sequenceid=417, compaction requested=false 2024-11-11T22:17:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 4c17d89df59d2cd5ce9ff2614859e6e4: 2024-11-11T22:17:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-11T22:17:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-11T22:17:27,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T22:17:27,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7640 sec 2024-11-11T22:17:27,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.7690 sec 2024-11-11T22:17:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T22:17:28,604 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1876 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1948 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 843 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2529 rows 2024-11-11T22:17:28,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 847 2024-11-11T22:17:28,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2541 rows 2024-11-11T22:17:28,605 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:17:28,605 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:17:28,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:28,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:28,606 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:28,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:17:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T22:17:28,620 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363448619"}]},"ts":"1731363448619"} 2024-11-11T22:17:28,622 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:17:28,622 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:17:28,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:17:28,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, UNASSIGN}] 2024-11-11T22:17:28,626 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, UNASSIGN 2024-11-11T22:17:28,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=4c17d89df59d2cd5ce9ff2614859e6e4, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:28,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, UNASSIGN because future has completed 2024-11-11T22:17:28,629 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:17:28,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:17:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T22:17:28,782 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:28,782 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:17:28,782 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 4c17d89df59d2cd5ce9ff2614859e6e4, disabling compactions & flushes 2024-11-11T22:17:28,782 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:28,782 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:28,782 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. after waiting 0 ms 2024-11-11T22:17:28,783 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:28,783 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 4c17d89df59d2cd5ce9ff2614859e6e4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T22:17:28,783 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=A 2024-11-11T22:17:28,783 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:28,783 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=B 2024-11-11T22:17:28,783 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:28,783 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4c17d89df59d2cd5ce9ff2614859e6e4, store=C 2024-11-11T22:17:28,783 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:28,791 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d4cef5b64b76455eb9c7145bb8134003_4c17d89df59d2cd5ce9ff2614859e6e4 is 50, key is test_row_0/A:col10/1731363446310/Put/seqid=0 2024-11-11T22:17:28,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742044_1220 (size=12454) 2024-11-11T22:17:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T22:17:29,202 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:29,206 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d4cef5b64b76455eb9c7145bb8134003_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d4cef5b64b76455eb9c7145bb8134003_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:29,207 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/54092439ab554f47906d18ac19c46e6b, store: [table=TestAcidGuarantees family=A region=4c17d89df59d2cd5ce9ff2614859e6e4] 2024-11-11T22:17:29,208 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/54092439ab554f47906d18ac19c46e6b is 175, key is test_row_0/A:col10/1731363446310/Put/seqid=0 2024-11-11T22:17:29,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742045_1221 (size=31255) 2024-11-11T22:17:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T22:17:29,614 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/54092439ab554f47906d18ac19c46e6b 2024-11-11T22:17:29,625 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/0874301a037a4461acd916c74d6a96a5 is 50, key is test_row_0/B:col10/1731363446310/Put/seqid=0 2024-11-11T22:17:29,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742046_1222 (size=12301) 2024-11-11T22:17:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T22:17:30,029 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/0874301a037a4461acd916c74d6a96a5 2024-11-11T22:17:30,037 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5f8c23d4a11a4871a400b264f222a530 is 50, key is test_row_0/C:col10/1731363446310/Put/seqid=0 2024-11-11T22:17:30,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742047_1223 (size=12301) 2024-11-11T22:17:30,441 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5f8c23d4a11a4871a400b264f222a530 2024-11-11T22:17:30,447 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/A/54092439ab554f47906d18ac19c46e6b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/54092439ab554f47906d18ac19c46e6b 2024-11-11T22:17:30,452 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/54092439ab554f47906d18ac19c46e6b, entries=150, sequenceid=428, filesize=30.5 K 2024-11-11T22:17:30,453 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/B/0874301a037a4461acd916c74d6a96a5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0874301a037a4461acd916c74d6a96a5 2024-11-11T22:17:30,459 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0874301a037a4461acd916c74d6a96a5, entries=150, sequenceid=428, filesize=12.0 K 2024-11-11T22:17:30,459 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/.tmp/C/5f8c23d4a11a4871a400b264f222a530 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f8c23d4a11a4871a400b264f222a530 2024-11-11T22:17:30,464 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f8c23d4a11a4871a400b264f222a530, entries=150, sequenceid=428, filesize=12.0 K 2024-11-11T22:17:30,465 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1682ms, sequenceid=428, compaction requested=true 2024-11-11T22:17:30,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f38349a882324dbf927adb62c307849c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d694cc7bb8f844449cef48d94483f7b6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/be6e7feb14a343e89fce07c8f2ac91cd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0723b8fb54374f7ca9533c7baae089e3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2ee0416cb224f07a8d90761053f719f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/485eba5424bb4ce8aa8da522a620c169, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/da712457c9a7443b87dfde8b74227531, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/11810bc6bd164157851b85466096b915, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/93084af1f75b4c009f73d7a7d3e4d736, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/66a41951d4ea46da99cf504f9460086b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9357dd684e6d48e1afb392eb642a2676, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b3fefb1301bf4ca1b064e95c94b9e0f6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/c4a18dfa63114e5c8a18271ac177a6fe, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/ba8eb70291bf44c487e7c42602402a51, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2d7b6a0b7f845509f278ac0a1d03abc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d8943ada5d854ad0a701d93c1c5b7d84, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2292ba9ab7646ddabd90c11cf205312, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f75cd587dba94cb0ae916503f30a549d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9fc04daf7f1c42ec84c404c7b2ff427c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/21565db96a5940b3a3c2394f26a4b575, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/3a58f01fdd9142e496400da5b41ad4e1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d592c8e597004e9f94c44bb0ee7acb81] to archive 2024-11-11T22:17:30,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:30,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f38349a882324dbf927adb62c307849c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f38349a882324dbf927adb62c307849c 2024-11-11T22:17:30,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d694cc7bb8f844449cef48d94483f7b6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d694cc7bb8f844449cef48d94483f7b6 2024-11-11T22:17:30,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/be6e7feb14a343e89fce07c8f2ac91cd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/be6e7feb14a343e89fce07c8f2ac91cd 2024-11-11T22:17:30,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0723b8fb54374f7ca9533c7baae089e3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0723b8fb54374f7ca9533c7baae089e3 2024-11-11T22:17:30,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2ee0416cb224f07a8d90761053f719f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2ee0416cb224f07a8d90761053f719f 2024-11-11T22:17:30,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/485eba5424bb4ce8aa8da522a620c169 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/485eba5424bb4ce8aa8da522a620c169 2024-11-11T22:17:30,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/da712457c9a7443b87dfde8b74227531 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/da712457c9a7443b87dfde8b74227531 2024-11-11T22:17:30,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/11810bc6bd164157851b85466096b915 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/11810bc6bd164157851b85466096b915 2024-11-11T22:17:30,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/93084af1f75b4c009f73d7a7d3e4d736 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/93084af1f75b4c009f73d7a7d3e4d736 2024-11-11T22:17:30,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/66a41951d4ea46da99cf504f9460086b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/66a41951d4ea46da99cf504f9460086b 2024-11-11T22:17:30,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9357dd684e6d48e1afb392eb642a2676 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9357dd684e6d48e1afb392eb642a2676 2024-11-11T22:17:30,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b3fefb1301bf4ca1b064e95c94b9e0f6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b3fefb1301bf4ca1b064e95c94b9e0f6 2024-11-11T22:17:30,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/c4a18dfa63114e5c8a18271ac177a6fe to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/c4a18dfa63114e5c8a18271ac177a6fe 2024-11-11T22:17:30,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/ba8eb70291bf44c487e7c42602402a51 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/ba8eb70291bf44c487e7c42602402a51 2024-11-11T22:17:30,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2d7b6a0b7f845509f278ac0a1d03abc to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2d7b6a0b7f845509f278ac0a1d03abc 2024-11-11T22:17:30,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d8943ada5d854ad0a701d93c1c5b7d84 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d8943ada5d854ad0a701d93c1c5b7d84 2024-11-11T22:17:30,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2292ba9ab7646ddabd90c11cf205312 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/b2292ba9ab7646ddabd90c11cf205312 2024-11-11T22:17:30,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f75cd587dba94cb0ae916503f30a549d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/f75cd587dba94cb0ae916503f30a549d 2024-11-11T22:17:30,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9fc04daf7f1c42ec84c404c7b2ff427c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/9fc04daf7f1c42ec84c404c7b2ff427c 2024-11-11T22:17:30,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/21565db96a5940b3a3c2394f26a4b575 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/21565db96a5940b3a3c2394f26a4b575 2024-11-11T22:17:30,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/3a58f01fdd9142e496400da5b41ad4e1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/3a58f01fdd9142e496400da5b41ad4e1 2024-11-11T22:17:30,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d592c8e597004e9f94c44bb0ee7acb81 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/d592c8e597004e9f94c44bb0ee7acb81 2024-11-11T22:17:30,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/5a8fd0c0b27c48b8b8edd67c190f6b59, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/1d9cd0e9dd374370b8ee085e16adf210, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/c7617e2c90a8475cafe87996f6ff944f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/eb990e229d114d73abac3670eee83faf, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/4db1a001b5cc4824be6a73de4e925e4f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/00471ac475fa41e99dd02442e9478a06, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/acc5943f2900451ca492c920b32141fa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/55aadd76087b4394b5fc1530634b5a1c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/8617934588e348b9951768945af2f755, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b2461b4d1af94d36829bba0411a5fa72, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/25de1f821660411cb24647d4d4a85136, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/cdb8c875872646dbaf0dedc78d74f170, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b4b97c0621374c34a529c48e83f51dde, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0c657d4188424d1b810bee29ebe01417, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/659378184bce4ea3aa21b300cfaa9e67, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/62e95ab778c841e0a87282bd4f9b3713, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/befc9d3192e345939895f56e6c20b256, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/599198ec7253479aa57de1ab64c3b218, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/34ae61a1ffc94456a916d9c4ee9b2105, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/26ab948200c144688015434e94c323a2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/14dd99255629481c9f2771fea5b932b6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/f5332decb60243638a44f1cdbf075c7a] to archive 2024-11-11T22:17:30,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:30,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/5a8fd0c0b27c48b8b8edd67c190f6b59 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/5a8fd0c0b27c48b8b8edd67c190f6b59 2024-11-11T22:17:30,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/1d9cd0e9dd374370b8ee085e16adf210 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/1d9cd0e9dd374370b8ee085e16adf210 2024-11-11T22:17:30,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/c7617e2c90a8475cafe87996f6ff944f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/c7617e2c90a8475cafe87996f6ff944f 2024-11-11T22:17:30,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/eb990e229d114d73abac3670eee83faf to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/eb990e229d114d73abac3670eee83faf 2024-11-11T22:17:30,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/4db1a001b5cc4824be6a73de4e925e4f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/4db1a001b5cc4824be6a73de4e925e4f 2024-11-11T22:17:30,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/00471ac475fa41e99dd02442e9478a06 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/00471ac475fa41e99dd02442e9478a06 2024-11-11T22:17:30,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/acc5943f2900451ca492c920b32141fa to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/acc5943f2900451ca492c920b32141fa 2024-11-11T22:17:30,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/55aadd76087b4394b5fc1530634b5a1c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/55aadd76087b4394b5fc1530634b5a1c 2024-11-11T22:17:30,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/8617934588e348b9951768945af2f755 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/8617934588e348b9951768945af2f755 2024-11-11T22:17:30,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b2461b4d1af94d36829bba0411a5fa72 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b2461b4d1af94d36829bba0411a5fa72 2024-11-11T22:17:30,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/25de1f821660411cb24647d4d4a85136 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/25de1f821660411cb24647d4d4a85136 2024-11-11T22:17:30,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/cdb8c875872646dbaf0dedc78d74f170 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/cdb8c875872646dbaf0dedc78d74f170 2024-11-11T22:17:30,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b4b97c0621374c34a529c48e83f51dde to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/b4b97c0621374c34a529c48e83f51dde 2024-11-11T22:17:30,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0c657d4188424d1b810bee29ebe01417 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0c657d4188424d1b810bee29ebe01417 2024-11-11T22:17:30,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/659378184bce4ea3aa21b300cfaa9e67 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/659378184bce4ea3aa21b300cfaa9e67 2024-11-11T22:17:30,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/62e95ab778c841e0a87282bd4f9b3713 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/62e95ab778c841e0a87282bd4f9b3713 2024-11-11T22:17:30,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/befc9d3192e345939895f56e6c20b256 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/befc9d3192e345939895f56e6c20b256 2024-11-11T22:17:30,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/599198ec7253479aa57de1ab64c3b218 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/599198ec7253479aa57de1ab64c3b218 2024-11-11T22:17:30,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/34ae61a1ffc94456a916d9c4ee9b2105 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/34ae61a1ffc94456a916d9c4ee9b2105 2024-11-11T22:17:30,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/26ab948200c144688015434e94c323a2 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/26ab948200c144688015434e94c323a2 2024-11-11T22:17:30,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/14dd99255629481c9f2771fea5b932b6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/14dd99255629481c9f2771fea5b932b6 2024-11-11T22:17:30,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/f5332decb60243638a44f1cdbf075c7a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/f5332decb60243638a44f1cdbf075c7a 2024-11-11T22:17:30,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5a4aba194fa04eaf86d460819b9bf550, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2324bbcad6764b3b8bee4e6de9cc2001, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/072f63adb9be4efd8d0f826c2ff7796e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/96937dfdfbec438b832f1a0a10153fd2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/f0e9470c8bf140fa8abfe2fce062e844, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c52a1602f01749459894b8ad18ec146c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2a9e89384530441a839b33cd57db23b3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7ba7a7a378b2468b9f582cb27860e3ae, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6c1a049d86954a389704a02976665e9d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9de2e1b05b434912884a8f9345cbbc5f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c984d349049944d4906f09bd93bed67b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7e4c7a04d0fb42e18bda0c6553e85fb1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9a670d6263c647b483635cfabbe8d864, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/21bc2ca80bf14821b4a7ef08ef8b8381, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/524dd7afe61c464faade92f276082045, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5742a85da0534d37aa2688550f244dc6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/a2183bf499a44eb4bd662eb9248ff91c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/3772cc22c99045b1baa0591cedc7de8b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/467f4d50fb49484cbfe219850bbf5599, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/aea4952d3e53466291ba267594e134c6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f5e16578b194070bb4ccb6463c2bd36, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6fca93905e1f4181aa0cf4252c076454] to archive 2024-11-11T22:17:30,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:30,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5a4aba194fa04eaf86d460819b9bf550 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5a4aba194fa04eaf86d460819b9bf550 2024-11-11T22:17:30,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2324bbcad6764b3b8bee4e6de9cc2001 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2324bbcad6764b3b8bee4e6de9cc2001 2024-11-11T22:17:30,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/072f63adb9be4efd8d0f826c2ff7796e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/072f63adb9be4efd8d0f826c2ff7796e 2024-11-11T22:17:30,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/96937dfdfbec438b832f1a0a10153fd2 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/96937dfdfbec438b832f1a0a10153fd2 2024-11-11T22:17:30,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/f0e9470c8bf140fa8abfe2fce062e844 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/f0e9470c8bf140fa8abfe2fce062e844 2024-11-11T22:17:30,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c52a1602f01749459894b8ad18ec146c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c52a1602f01749459894b8ad18ec146c 2024-11-11T22:17:30,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2a9e89384530441a839b33cd57db23b3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/2a9e89384530441a839b33cd57db23b3 2024-11-11T22:17:30,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7ba7a7a378b2468b9f582cb27860e3ae to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7ba7a7a378b2468b9f582cb27860e3ae 2024-11-11T22:17:30,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6c1a049d86954a389704a02976665e9d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6c1a049d86954a389704a02976665e9d 2024-11-11T22:17:30,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9de2e1b05b434912884a8f9345cbbc5f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9de2e1b05b434912884a8f9345cbbc5f 2024-11-11T22:17:30,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c984d349049944d4906f09bd93bed67b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/c984d349049944d4906f09bd93bed67b 2024-11-11T22:17:30,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7e4c7a04d0fb42e18bda0c6553e85fb1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7e4c7a04d0fb42e18bda0c6553e85fb1 2024-11-11T22:17:30,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9a670d6263c647b483635cfabbe8d864 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/9a670d6263c647b483635cfabbe8d864 2024-11-11T22:17:30,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/21bc2ca80bf14821b4a7ef08ef8b8381 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/21bc2ca80bf14821b4a7ef08ef8b8381 2024-11-11T22:17:30,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/524dd7afe61c464faade92f276082045 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/524dd7afe61c464faade92f276082045 2024-11-11T22:17:30,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5742a85da0534d37aa2688550f244dc6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5742a85da0534d37aa2688550f244dc6 2024-11-11T22:17:30,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/a2183bf499a44eb4bd662eb9248ff91c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/a2183bf499a44eb4bd662eb9248ff91c 2024-11-11T22:17:30,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/3772cc22c99045b1baa0591cedc7de8b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/3772cc22c99045b1baa0591cedc7de8b 2024-11-11T22:17:30,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/467f4d50fb49484cbfe219850bbf5599 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/467f4d50fb49484cbfe219850bbf5599 2024-11-11T22:17:30,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/aea4952d3e53466291ba267594e134c6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/aea4952d3e53466291ba267594e134c6 2024-11-11T22:17:30,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f5e16578b194070bb4ccb6463c2bd36 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f5e16578b194070bb4ccb6463c2bd36 2024-11-11T22:17:30,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6fca93905e1f4181aa0cf4252c076454 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/6fca93905e1f4181aa0cf4252c076454 2024-11-11T22:17:30,575 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=4 2024-11-11T22:17:30,576 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. 2024-11-11T22:17:30,576 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 4c17d89df59d2cd5ce9ff2614859e6e4: Waiting for close lock at 1731363448782Running coprocessor pre-close hooks at 1731363448782Disabling compacts and flushes for region at 1731363448782Disabling writes for close at 1731363448783 (+1 ms)Obtaining lock to block concurrent updates at 1731363448783Preparing flush snapshotting stores in 4c17d89df59d2cd5ce9ff2614859e6e4 at 1731363448783Finished memstore snapshotting TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731363448783Flushing stores of TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4. at 1731363448784 (+1 ms)Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/A: creating writer at 1731363448784Flushing mob file 4c17d89df59d2cd5ce9ff2614859e6e4/A: appending metadata at 1731363448791 (+7 ms)Flushing mob file 4c17d89df59d2cd5ce9ff2614859e6e4/A: closing flushed file at 1731363448791Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/A: appending metadata at 1731363449207 (+416 ms)Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/A: closing flushed file at 1731363449207Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/B: creating writer at 1731363449621 (+414 ms)Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/B: appending metadata at 1731363449624 (+3 ms)Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/B: closing flushed file at 1731363449624Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/C: creating writer at 1731363450033 (+409 ms)Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/C: appending metadata at 1731363450036 (+3 ms)Flushing 4c17d89df59d2cd5ce9ff2614859e6e4/C: closing flushed file at 1731363450036Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7508bcc: reopening flushed file at 1731363450446 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ed4ec79: reopening flushed file at 1731363450452 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@190f2b3: reopening flushed file at 1731363450459 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4c17d89df59d2cd5ce9ff2614859e6e4 in 1682ms, sequenceid=428, compaction requested=true at 1731363450465 (+6 ms)Writing region close event to WAL at 1731363450572 (+107 ms)Running coprocessor post-close hooks at 1731363450576 (+4 ms)Closed at 1731363450576 2024-11-11T22:17:30,578 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,579 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=4c17d89df59d2cd5ce9ff2614859e6e4, regionState=CLOSED 2024-11-11T22:17:30,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:17:30,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T22:17:30,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c17d89df59d2cd5ce9ff2614859e6e4, server=8d82b89dd27d,36911,1731363387997 in 1.9520 sec 2024-11-11T22:17:30,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-11T22:17:30,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c17d89df59d2cd5ce9ff2614859e6e4, UNASSIGN in 1.9570 sec 2024-11-11T22:17:30,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T22:17:30,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9610 sec 2024-11-11T22:17:30,587 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363450587"}]},"ts":"1731363450587"} 2024-11-11T22:17:30,590 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:17:30,590 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:17:30,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9830 sec 2024-11-11T22:17:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T22:17:30,769 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:30,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:17:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:30,771 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:30,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T22:17:30,772 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:30,775 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,777 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/recovered.edits] 2024-11-11T22:17:30,779 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0e0e262c929f45908247beae79d7e185 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/0e0e262c929f45908247beae79d7e185 2024-11-11T22:17:30,781 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/54092439ab554f47906d18ac19c46e6b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/54092439ab554f47906d18ac19c46e6b 2024-11-11T22:17:30,782 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/e7892c11b5ed4c42b4d9b6e8b64fbd9b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/A/e7892c11b5ed4c42b4d9b6e8b64fbd9b 2024-11-11T22:17:30,785 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0874301a037a4461acd916c74d6a96a5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/0874301a037a4461acd916c74d6a96a5 2024-11-11T22:17:30,786 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/43323b33bdf445ffb365b067b7412ee8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/43323b33bdf445ffb365b067b7412ee8 2024-11-11T22:17:30,787 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/6bb6af79fc874db9b532b7e28934e42a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/B/6bb6af79fc874db9b532b7e28934e42a 2024-11-11T22:17:30,790 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/421dae5922c14976b844bfdeab02cf86 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/421dae5922c14976b844bfdeab02cf86 2024-11-11T22:17:30,791 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f8c23d4a11a4871a400b264f222a530 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/5f8c23d4a11a4871a400b264f222a530 2024-11-11T22:17:30,792 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7b6d9968dde74d368b9e78df4fbff310 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/C/7b6d9968dde74d368b9e78df4fbff310 2024-11-11T22:17:30,795 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/recovered.edits/431.seqid to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4/recovered.edits/431.seqid 2024-11-11T22:17:30,796 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,796 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:17:30,796 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:17:30,797 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T22:17:30,801 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111118ae9d613934ef6a7668e1153ec5adc_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111118ae9d613934ef6a7668e1153ec5adc_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,802 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112026f68fc0f74fa9987f558134fb4098_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112026f68fc0f74fa9987f558134fb4098_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,803 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113444d95f3c8843c0a7fb6344210de9a5_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113444d95f3c8843c0a7fb6344210de9a5_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,805 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c776981f4ef402a82dbac2398913153_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c776981f4ef402a82dbac2398913153_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,806 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116a549d14ae654695b74decd618670dcd_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116a549d14ae654695b74decd618670dcd_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,807 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d60d8d08be047ce896bc308a61a7de0_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d60d8d08be047ce896bc308a61a7de0_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,809 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d787aa6d41244ab9eb1640aeafb6195_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116d787aa6d41244ab9eb1640aeafb6195_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,810 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111892989b19661488c81ecee862a16745d_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111892989b19661488c81ecee862a16745d_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,811 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119959f52467294fabb5b77749940d772d_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119959f52467294fabb5b77749940d772d_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,813 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119b2a4737d3af4750999f8c2720a8efa6_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119b2a4737d3af4750999f8c2720a8efa6_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,814 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119e95093e9e54432292e45030122fb563_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119e95093e9e54432292e45030122fb563_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,815 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a6eee0c551ac461aa0ac221503aced25_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a6eee0c551ac461aa0ac221503aced25_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,816 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a8dd57e2493848d68152054a381984a1_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a8dd57e2493848d68152054a381984a1_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,817 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b9088347ee9a41af831e33eab4d90710_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b9088347ee9a41af831e33eab4d90710_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,818 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d39ed280eb0f45d6832395563882fe7d_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d39ed280eb0f45d6832395563882fe7d_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,819 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d4cef5b64b76455eb9c7145bb8134003_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d4cef5b64b76455eb9c7145bb8134003_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,820 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0afaab63b8a45a0bb78d82f8cd5af5b_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e0afaab63b8a45a0bb78d82f8cd5af5b_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,821 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e33a71b022b444c6aed5f7bae998e330_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e33a71b022b444c6aed5f7bae998e330_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,822 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ead602345795415faf174a8d7fd47d92_4c17d89df59d2cd5ce9ff2614859e6e4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ead602345795415faf174a8d7fd47d92_4c17d89df59d2cd5ce9ff2614859e6e4 2024-11-11T22:17:30,823 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:17:30,825 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:30,828 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:17:30,830 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:17:30,833 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:30,834 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:17:30,834 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731363450834"}]},"ts":"9223372036854775807"} 2024-11-11T22:17:30,837 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:17:30,837 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4c17d89df59d2cd5ce9ff2614859e6e4, NAME => 'TestAcidGuarantees,,1731363421969.4c17d89df59d2cd5ce9ff2614859e6e4.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:17:30,837 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:17:30,837 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731363450837"}]},"ts":"9223372036854775807"} 2024-11-11T22:17:30,840 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:17:30,841 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:30,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-11T22:17:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T22:17:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:17:30,880 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:30,891 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1014836282_22 at /127.0.0.1:47810 [Waiting for operation #343] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1165202873_22 at /127.0.0.1:41912 [Waiting for operation #318] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1014836282_22 at /127.0.0.1:42032 [Waiting for operation #251] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=780 (was 640) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1933 (was 1595) - AvailableMemoryMB LEAK? - 2024-11-11T22:17:30,905 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=780, ProcessCount=11, AvailableMemoryMB=1932 2024-11-11T22:17:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:17:30,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:17:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:30,909 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:17:30,909 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:30,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-11T22:17:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T22:17:30,910 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:17:30,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742048_1224 (size=1061) 2024-11-11T22:17:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T22:17:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T22:17:31,318 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e 2024-11-11T22:17:31,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742049_1225 (size=53) 2024-11-11T22:17:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T22:17:31,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:17:31,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aea091a0b91074e84f623acef361ebf8, disabling compactions & flushes 2024-11-11T22:17:31,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:31,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:31,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. after waiting 0 ms 2024-11-11T22:17:31,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:31,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:31,725 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aea091a0b91074e84f623acef361ebf8: Waiting for close lock at 1731363451725Disabling compacts and flushes for region at 1731363451725Disabling writes for close at 1731363451725Writing region close event to WAL at 1731363451725Closed at 1731363451725 2024-11-11T22:17:31,727 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:17:31,727 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363451727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363451727"}]},"ts":"1731363451727"} 2024-11-11T22:17:31,730 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:17:31,732 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:17:31,733 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363451732"}]},"ts":"1731363451732"} 2024-11-11T22:17:31,735 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:17:31,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, ASSIGN}] 2024-11-11T22:17:31,737 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, ASSIGN 2024-11-11T22:17:31,737 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, ASSIGN; state=OFFLINE, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=false 2024-11-11T22:17:31,888 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=aea091a0b91074e84f623acef361ebf8, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:31,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, ASSIGN because future has completed 2024-11-11T22:17:31,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:17:32,047 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:32,047 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:17:32,047 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,047 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:17:32,047 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,047 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,049 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T22:17:32,050 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:32,050 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea091a0b91074e84f623acef361ebf8 columnFamilyName A 2024-11-11T22:17:32,050 DEBUG [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:32,051 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.HStore(327): Store=aea091a0b91074e84f623acef361ebf8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:32,051 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,052 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:32,052 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea091a0b91074e84f623acef361ebf8 columnFamilyName B 2024-11-11T22:17:32,052 DEBUG [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:32,053 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.HStore(327): Store=aea091a0b91074e84f623acef361ebf8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:32,053 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,054 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:32,054 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aea091a0b91074e84f623acef361ebf8 columnFamilyName C 2024-11-11T22:17:32,054 DEBUG [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:32,055 INFO [StoreOpener-aea091a0b91074e84f623acef361ebf8-1 {}] regionserver.HStore(327): Store=aea091a0b91074e84f623acef361ebf8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:32,055 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:32,055 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,055 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,056 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,057 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,057 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,057 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:17:32,058 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,060 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:17:32,061 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened aea091a0b91074e84f623acef361ebf8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58782424, jitterRate=-0.12407362461090088}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:17:32,061 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:32,062 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for aea091a0b91074e84f623acef361ebf8: Running coprocessor pre-open hook at 1731363452048Writing region info on filesystem at 1731363452048Initializing all the Stores at 1731363452048Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363452048Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363452049 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363452049Cleaning up temporary data from old regions at 1731363452057 (+8 ms)Running coprocessor post-open hooks at 1731363452061 (+4 ms)Region opened successfully at 1731363452062 (+1 ms) 2024-11-11T22:17:32,062 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., pid=65, masterSystemTime=1731363452044 2024-11-11T22:17:32,064 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:32,064 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:32,065 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=aea091a0b91074e84f623acef361ebf8, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:32,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:17:32,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T22:17:32,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 in 176 msec 2024-11-11T22:17:32,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-11T22:17:32,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, ASSIGN in 334 msec 2024-11-11T22:17:32,071 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:17:32,071 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363452071"}]},"ts":"1731363452071"} 2024-11-11T22:17:32,073 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:17:32,073 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:17:32,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-11T22:17:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-11T22:17:33,060 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:33,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31dd285b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,062 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d66d54e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4ed6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,064 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:17:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,41919,1731363387169 2024-11-11T22:17:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58b197a6 2024-11-11T22:17:33,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:17:33,070 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:17:33,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19506d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,072 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa6e67f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,074 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd25d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0e6418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,077 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5a83bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,078 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d65a854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e58dfe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,080 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c974ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,081 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad5a418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48668fa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,083 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fccbbb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,084 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79039e05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b4c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,086 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653c3a9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,087 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3acd3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab276b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,088 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,089 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34775e84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,090 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52afdc36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@481e04ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,091 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f92084, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,093 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6124c0a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705214bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,095 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae3e7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,096 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195309e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2c14f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,098 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38020ce0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,100 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f20fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bdde348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:17:33,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:17:33,102 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:17:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:17:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:17:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa40e51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:17:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:17:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:33,103 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:17:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707755f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:17:33,104 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,104 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,104 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,105 DEBUG [Thread-1066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:33,105 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,105 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,106 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,106 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,106 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,106 DEBUG [Thread-1064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:17:33,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T22:17:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:17:33,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:17:33,110 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:17:33,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:17:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,113 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:33,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:17:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:33,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:17:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:33,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:33,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/96e52b36a36f4d07af5f5478e57ddc9c is 50, key is test_row_0/A:col10/1731363453120/Put/seqid=0 2024-11-11T22:17:33,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742050_1226 (size=12001) 2024-11-11T22:17:33,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363513166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363513175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363513179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363513180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363513181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:17:33,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:33,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363513290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363513290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363513290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363513290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363513290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:33,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:17:33,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363513499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363513499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363513499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363513510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363513511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/96e52b36a36f4d07af5f5478e57ddc9c 2024-11-11T22:17:33,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/35cf2ef957014d67ac77b34d0c798702 is 50, key is test_row_0/B:col10/1731363453120/Put/seqid=0 2024-11-11T22:17:33,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742051_1227 (size=12001) 2024-11-11T22:17:33,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/35cf2ef957014d67ac77b34d0c798702 2024-11-11T22:17:33,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/9452793a28e34cc4a95289f4f67c1581 is 50, key is test_row_0/C:col10/1731363453120/Put/seqid=0 2024-11-11T22:17:33,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742052_1228 (size=12001) 2024-11-11T22:17:33,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/9452793a28e34cc4a95289f4f67c1581 2024-11-11T22:17:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/96e52b36a36f4d07af5f5478e57ddc9c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/96e52b36a36f4d07af5f5478e57ddc9c 2024-11-11T22:17:33,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:17:33,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/96e52b36a36f4d07af5f5478e57ddc9c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:17:33,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/35cf2ef957014d67ac77b34d0c798702 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/35cf2ef957014d67ac77b34d0c798702 2024-11-11T22:17:33,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/35cf2ef957014d67ac77b34d0c798702, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:17:33,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/9452793a28e34cc4a95289f4f67c1581 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/9452793a28e34cc4a95289f4f67c1581 2024-11-11T22:17:33,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/9452793a28e34cc4a95289f4f67c1581, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T22:17:33,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for aea091a0b91074e84f623acef361ebf8 in 661ms, sequenceid=13, compaction requested=false 2024-11-11T22:17:33,782 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T22:17:33,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363453152, max timestamp=1731363453813Num uniques 859; 2024-11-11T22:17:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363453152, max timestamp=1731363453813Num uniques 859; 2024-11-11T22:17:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363453152, max timestamp=1731363453813Num uniques 859; 2024-11-11T22:17:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:33,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:33,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-11T22:17:33,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:33,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:33,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:33,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:33,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:33,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:33,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/acbec3bdb3a147a7942425b8a03b8c60 is 50, key is test_row_0/A:col10/1731363453829/Put/seqid=0 2024-11-11T22:17:33,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:33,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363453860, max timestamp=1731363453893Num uniques 859; 2024-11-11T22:17:33,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:33,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363453860, max timestamp=1731363453894Num uniques 859; 2024-11-11T22:17:33,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363453860, max timestamp=1731363453893Num uniques 859; 2024-11-11T22:17:33,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:33,898 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:33,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363513881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742053_1229 (size=19021) 2024-11-11T22:17:33,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/acbec3bdb3a147a7942425b8a03b8c60 2024-11-11T22:17:33,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf is 50, key is test_row_0/B:col10/1731363453829/Put/seqid=0 2024-11-11T22:17:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363513958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363513966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363513966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:33,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363513970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:33,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363514010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742054_1230 (size=12001) 2024-11-11T22:17:34,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf 2024-11-11T22:17:34,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/7890f1c50dee4ad1b43b91e5929b34cf is 50, key is test_row_0/C:col10/1731363453829/Put/seqid=0 2024-11-11T22:17:34,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:34,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:34,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363514080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363514080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363514080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363514080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742055_1231 (size=12001) 2024-11-11T22:17:34,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:34,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:34,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:34,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:34,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363514219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:17:34,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363514290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363514290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363514290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363514290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:34,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:34,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/7890f1c50dee4ad1b43b91e5929b34cf 2024-11-11T22:17:34,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/acbec3bdb3a147a7942425b8a03b8c60 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/acbec3bdb3a147a7942425b8a03b8c60 2024-11-11T22:17:34,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/acbec3bdb3a147a7942425b8a03b8c60, entries=300, sequenceid=39, filesize=18.6 K 2024-11-11T22:17:34,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf 2024-11-11T22:17:34,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf, entries=150, sequenceid=39, filesize=11.7 K 2024-11-11T22:17:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/7890f1c50dee4ad1b43b91e5929b34cf as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/7890f1c50dee4ad1b43b91e5929b34cf 2024-11-11T22:17:34,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/7890f1c50dee4ad1b43b91e5929b34cf, entries=150, sequenceid=39, filesize=11.7 K 2024-11-11T22:17:34,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for aea091a0b91074e84f623acef361ebf8 in 690ms, sequenceid=39, compaction requested=false 2024-11-11T22:17:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:34,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T22:17:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:34,523 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T22:17:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:34,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:34,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/c434bc9f4c2f416cb1dee8fb3d5f0a02 is 50, key is test_row_0/A:col10/1731363453955/Put/seqid=0 2024-11-11T22:17:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:34,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:34,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742056_1232 (size=12001) 2024-11-11T22:17:34,556 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/c434bc9f4c2f416cb1dee8fb3d5f0a02 2024-11-11T22:17:34,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/5e30f69d6368485099a159be827babb4 is 50, key is test_row_0/B:col10/1731363453955/Put/seqid=0 2024-11-11T22:17:34,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742057_1233 (size=12001) 2024-11-11T22:17:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363514593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363514599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363514600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363514600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363514610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363514710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363514919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:34,988 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/5e30f69d6368485099a159be827babb4 2024-11-11T22:17:34,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/4f92a30b84574f729cf05d8eb9149a78 is 50, key is test_row_0/C:col10/1731363453955/Put/seqid=0 2024-11-11T22:17:35,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742058_1234 (size=12001) 2024-11-11T22:17:35,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363515109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:35,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363515109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363515120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363515120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:35,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:35,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363515229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:17:35,401 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/4f92a30b84574f729cf05d8eb9149a78 2024-11-11T22:17:35,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/c434bc9f4c2f416cb1dee8fb3d5f0a02 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c434bc9f4c2f416cb1dee8fb3d5f0a02 2024-11-11T22:17:35,409 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c434bc9f4c2f416cb1dee8fb3d5f0a02, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T22:17:35,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/5e30f69d6368485099a159be827babb4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/5e30f69d6368485099a159be827babb4 2024-11-11T22:17:35,415 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/5e30f69d6368485099a159be827babb4, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T22:17:35,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/4f92a30b84574f729cf05d8eb9149a78 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4f92a30b84574f729cf05d8eb9149a78 2024-11-11T22:17:35,421 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4f92a30b84574f729cf05d8eb9149a78, entries=150, sequenceid=75, filesize=11.7 K 2024-11-11T22:17:35,423 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for aea091a0b91074e84f623acef361ebf8 in 900ms, sequenceid=75, compaction requested=true 2024-11-11T22:17:35,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:35,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:35,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T22:17:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T22:17:35,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T22:17:35,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3120 sec 2024-11-11T22:17:35,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.3230 sec 2024-11-11T22:17:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:35,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:17:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:35,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:35,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/c421ae42e9514218ad3f8ceab77a31cb is 50, key is test_row_0/A:col10/1731363454556/Put/seqid=0 2024-11-11T22:17:35,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742059_1235 (size=12001) 2024-11-11T22:17:35,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/c421ae42e9514218ad3f8ceab77a31cb 2024-11-11T22:17:35,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7bb6f162a45b4a3e9fc180a4aca210df is 50, key is test_row_0/B:col10/1731363454556/Put/seqid=0 2024-11-11T22:17:35,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742060_1236 (size=12001) 2024-11-11T22:17:35,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363515839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:35,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363515949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:36,103 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:17:36,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363516120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:36,126 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_0' in aea091a0b91074e84f623acef361ebf8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:36,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363516129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:36,132 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in aea091a0b91074e84f623acef361ebf8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363516139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363516139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:36,144 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_1' in aea091a0b91074e84f623acef361ebf8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:36,144 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in aea091a0b91074e84f623acef361ebf8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:36,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363516159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:36,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7bb6f162a45b4a3e9fc180a4aca210df 2024-11-11T22:17:36,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/d71bc205ea4f49f085e6d1a8de761fc1 is 50, key is test_row_0/C:col10/1731363454556/Put/seqid=0 2024-11-11T22:17:36,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742061_1237 (size=12001) 2024-11-11T22:17:36,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363516469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:36,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:36,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/d71bc205ea4f49f085e6d1a8de761fc1 2024-11-11T22:17:36,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/c421ae42e9514218ad3f8ceab77a31cb as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c421ae42e9514218ad3f8ceab77a31cb 2024-11-11T22:17:36,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c421ae42e9514218ad3f8ceab77a31cb, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:17:36,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7bb6f162a45b4a3e9fc180a4aca210df as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7bb6f162a45b4a3e9fc180a4aca210df 2024-11-11T22:17:36,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7bb6f162a45b4a3e9fc180a4aca210df, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:17:36,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/d71bc205ea4f49f085e6d1a8de761fc1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d71bc205ea4f49f085e6d1a8de761fc1 2024-11-11T22:17:36,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d71bc205ea4f49f085e6d1a8de761fc1, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T22:17:36,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for aea091a0b91074e84f623acef361ebf8 in 915ms, sequenceid=87, compaction requested=true 2024-11-11T22:17:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:36,657 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:36,657 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:36,659 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:36,659 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:36,659 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:36,659 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/35cf2ef957014d67ac77b34d0c798702, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/5e30f69d6368485099a159be827babb4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7bb6f162a45b4a3e9fc180a4aca210df] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=46.9 K 2024-11-11T22:17:36,660 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 35cf2ef957014d67ac77b34d0c798702, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363453118 2024-11-11T22:17:36,661 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c7e3c9bc8f47e39dfb5fb3c5bcb4cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731363453818 2024-11-11T22:17:36,661 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:36,661 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:36,661 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:36,661 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/96e52b36a36f4d07af5f5478e57ddc9c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/acbec3bdb3a147a7942425b8a03b8c60, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c434bc9f4c2f416cb1dee8fb3d5f0a02, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c421ae42e9514218ad3f8ceab77a31cb] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=53.7 K 2024-11-11T22:17:36,662 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e52b36a36f4d07af5f5478e57ddc9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363453118 2024-11-11T22:17:36,662 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e30f69d6368485099a159be827babb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731363453955 2024-11-11T22:17:36,664 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting acbec3bdb3a147a7942425b8a03b8c60, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731363453152 2024-11-11T22:17:36,664 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bb6f162a45b4a3e9fc180a4aca210df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363454556 2024-11-11T22:17:36,665 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c434bc9f4c2f416cb1dee8fb3d5f0a02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731363453955 2024-11-11T22:17:36,666 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c421ae42e9514218ad3f8ceab77a31cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363454556 2024-11-11T22:17:36,715 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:36,716 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/0677e50024a74d67bb4794bbd9640f5f is 50, key is test_row_0/B:col10/1731363454556/Put/seqid=0 2024-11-11T22:17:36,719 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:36,720 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/63734e21f5da410aa14f5aeefa375536 is 50, key is test_row_0/A:col10/1731363454556/Put/seqid=0 2024-11-11T22:17:36,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742062_1238 (size=12139) 2024-11-11T22:17:36,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742063_1239 (size=12139) 2024-11-11T22:17:36,755 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/63734e21f5da410aa14f5aeefa375536 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/63734e21f5da410aa14f5aeefa375536 2024-11-11T22:17:36,765 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into 63734e21f5da410aa14f5aeefa375536(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:36,765 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:36,765 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=12, startTime=1731363456657; duration=0sec 2024-11-11T22:17:36,765 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:36,765 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:36,765 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:36,767 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:36,767 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:36,767 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:36,767 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/9452793a28e34cc4a95289f4f67c1581, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/7890f1c50dee4ad1b43b91e5929b34cf, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4f92a30b84574f729cf05d8eb9149a78, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d71bc205ea4f49f085e6d1a8de761fc1] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=46.9 K 2024-11-11T22:17:36,768 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9452793a28e34cc4a95289f4f67c1581, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731363453118 2024-11-11T22:17:36,769 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7890f1c50dee4ad1b43b91e5929b34cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731363453818 2024-11-11T22:17:36,771 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f92a30b84574f729cf05d8eb9149a78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731363453955 2024-11-11T22:17:36,772 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d71bc205ea4f49f085e6d1a8de761fc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363454556 2024-11-11T22:17:36,806 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#197 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:36,809 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/63000643abe44e90b49d0f8dab0b34b1 is 50, key is test_row_0/C:col10/1731363454556/Put/seqid=0 2024-11-11T22:17:36,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742064_1240 (size=12139) 2024-11-11T22:17:36,833 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/63000643abe44e90b49d0f8dab0b34b1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/63000643abe44e90b49d0f8dab0b34b1 2024-11-11T22:17:36,850 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into 63000643abe44e90b49d0f8dab0b34b1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:36,850 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:36,850 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=12, startTime=1731363456657; duration=0sec 2024-11-11T22:17:36,851 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:36,851 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:36,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363455817, max timestamp=1731363456991Num uniques 859; 2024-11-11T22:17:36,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363455817, max timestamp=1731363456991Num uniques 859; 2024-11-11T22:17:36,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:36,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:36,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-11T22:17:36,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:36,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:36,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:36,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:36,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:36,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:37,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/1cf44b5dafbf489db2ef1be8e7955db3 is 50, key is test_row_0/A:col10/1731363455822/Put/seqid=0 2024-11-11T22:17:37,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742065_1241 (size=14341) 2024-11-11T22:17:37,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/1cf44b5dafbf489db2ef1be8e7955db3 2024-11-11T22:17:37,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/9dc6fbe5ca9e4b058663e18c9c75f016 is 50, key is test_row_0/B:col10/1731363455822/Put/seqid=0 2024-11-11T22:17:37,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742066_1242 (size=12001) 2024-11-11T22:17:37,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/9dc6fbe5ca9e4b058663e18c9c75f016 2024-11-11T22:17:37,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ee9909a346ae46aaa0c34a85ca679744 is 50, key is test_row_0/C:col10/1731363455822/Put/seqid=0 2024-11-11T22:17:37,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742067_1243 (size=12001) 2024-11-11T22:17:37,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ee9909a346ae46aaa0c34a85ca679744 2024-11-11T22:17:37,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/1cf44b5dafbf489db2ef1be8e7955db3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1cf44b5dafbf489db2ef1be8e7955db3 2024-11-11T22:17:37,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1cf44b5dafbf489db2ef1be8e7955db3, entries=200, sequenceid=110, filesize=14.0 K 2024-11-11T22:17:37,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/9dc6fbe5ca9e4b058663e18c9c75f016 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/9dc6fbe5ca9e4b058663e18c9c75f016 2024-11-11T22:17:37,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/9dc6fbe5ca9e4b058663e18c9c75f016, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T22:17:37,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ee9909a346ae46aaa0c34a85ca679744 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ee9909a346ae46aaa0c34a85ca679744 2024-11-11T22:17:37,142 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/0677e50024a74d67bb4794bbd9640f5f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0677e50024a74d67bb4794bbd9640f5f 2024-11-11T22:17:37,158 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into 0677e50024a74d67bb4794bbd9640f5f(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:37,158 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:37,158 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=12, startTime=1731363456657; duration=0sec 2024-11-11T22:17:37,158 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:37,158 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:37,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ee9909a346ae46aaa0c34a85ca679744, entries=150, sequenceid=110, filesize=11.7 K 2024-11-11T22:17:37,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=107.34 KB/109920 for aea091a0b91074e84f623acef361ebf8 in 164ms, sequenceid=110, compaction requested=false 2024-11-11T22:17:37,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:37,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:17:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/b1d5e0bf3f234375ada307085ac9e0a2 is 50, key is test_row_0/A:col10/1731363457162/Put/seqid=0 2024-11-11T22:17:37,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742068_1244 (size=14391) 2024-11-11T22:17:37,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/b1d5e0bf3f234375ada307085ac9e0a2 2024-11-11T22:17:37,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/51c8033a773046308fe4173213f33639 is 50, key is test_row_0/B:col10/1731363457162/Put/seqid=0 2024-11-11T22:17:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742069_1245 (size=12051) 2024-11-11T22:17:37,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363517215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:37,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/51c8033a773046308fe4173213f33639 2024-11-11T22:17:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:37,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/20c2856b199e44e2b9ee4ca546b92bf8 is 50, key is test_row_0/C:col10/1731363457162/Put/seqid=0 2024-11-11T22:17:37,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742070_1246 (size=12051) 2024-11-11T22:17:37,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/20c2856b199e44e2b9ee4ca546b92bf8 2024-11-11T22:17:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/b1d5e0bf3f234375ada307085ac9e0a2 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b1d5e0bf3f234375ada307085ac9e0a2 2024-11-11T22:17:37,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b1d5e0bf3f234375ada307085ac9e0a2, entries=200, sequenceid=131, filesize=14.1 K 2024-11-11T22:17:37,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/51c8033a773046308fe4173213f33639 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/51c8033a773046308fe4173213f33639 2024-11-11T22:17:37,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/51c8033a773046308fe4173213f33639, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T22:17:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/20c2856b199e44e2b9ee4ca546b92bf8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/20c2856b199e44e2b9ee4ca546b92bf8 2024-11-11T22:17:37,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/20c2856b199e44e2b9ee4ca546b92bf8, entries=150, sequenceid=131, filesize=11.8 K 2024-11-11T22:17:37,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for aea091a0b91074e84f623acef361ebf8 in 115ms, sequenceid=131, compaction requested=true 2024-11-11T22:17:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:37,278 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:37,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:37,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:37,279 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:37,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:37,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T22:17:37,279 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:37,280 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:37,280 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:37,280 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:37,280 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,280 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/63734e21f5da410aa14f5aeefa375536, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1cf44b5dafbf489db2ef1be8e7955db3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b1d5e0bf3f234375ada307085ac9e0a2] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=39.9 K 2024-11-11T22:17:37,281 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:37,281 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:37,281 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,281 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0677e50024a74d67bb4794bbd9640f5f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/9dc6fbe5ca9e4b058663e18c9c75f016, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/51c8033a773046308fe4173213f33639] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=35.3 K 2024-11-11T22:17:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T22:17:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:17:37,283 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:37,284 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63734e21f5da410aa14f5aeefa375536, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363454556 2024-11-11T22:17:37,285 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:37,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:37,286 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0677e50024a74d67bb4794bbd9640f5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363454556 2024-11-11T22:17:37,286 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf44b5dafbf489db2ef1be8e7955db3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731363455817 2024-11-11T22:17:37,287 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dc6fbe5ca9e4b058663e18c9c75f016, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731363455822 2024-11-11T22:17:37,288 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1d5e0bf3f234375ada307085ac9e0a2, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731363457132 2024-11-11T22:17:37,288 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c8033a773046308fe4173213f33639, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731363457140 2024-11-11T22:17:37,312 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:37,313 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/23e16f1d817544328f11703f3d2ea162 is 50, key is test_row_0/B:col10/1731363457162/Put/seqid=0 2024-11-11T22:17:37,315 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:37,315 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/cc5ccdcee13946eeb6db29586d79f823 is 50, key is test_row_0/A:col10/1731363457162/Put/seqid=0 2024-11-11T22:17:37,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:17:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:37,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:37,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742072_1248 (size=12291) 2024-11-11T22:17:37,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742071_1247 (size=12291) 2024-11-11T22:17:37,371 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/cc5ccdcee13946eeb6db29586d79f823 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cc5ccdcee13946eeb6db29586d79f823 2024-11-11T22:17:37,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/1716672cce5c41f3a1ae624778a28180 is 50, key is test_row_0/A:col10/1731363457202/Put/seqid=0 2024-11-11T22:17:37,380 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into cc5ccdcee13946eeb6db29586d79f823(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:37,380 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:37,380 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=13, startTime=1731363457278; duration=0sec 2024-11-11T22:17:37,380 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:37,380 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:37,380 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:37,385 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:37,385 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:37,385 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,385 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/63000643abe44e90b49d0f8dab0b34b1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ee9909a346ae46aaa0c34a85ca679744, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/20c2856b199e44e2b9ee4ca546b92bf8] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=35.3 K 2024-11-11T22:17:37,386 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63000643abe44e90b49d0f8dab0b34b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731363454556 2024-11-11T22:17:37,386 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee9909a346ae46aaa0c34a85ca679744, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731363455822 2024-11-11T22:17:37,387 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20c2856b199e44e2b9ee4ca546b92bf8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731363457140 2024-11-11T22:17:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:17:37,404 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:37,405 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/3474dc378d7e46b6b8102377b6d4c6b9 is 50, key is test_row_0/C:col10/1731363457162/Put/seqid=0 2024-11-11T22:17:37,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742073_1249 (size=14541) 2024-11-11T22:17:37,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/1716672cce5c41f3a1ae624778a28180 2024-11-11T22:17:37,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742074_1250 (size=12291) 2024-11-11T22:17:37,436 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/3474dc378d7e46b6b8102377b6d4c6b9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3474dc378d7e46b6b8102377b6d4c6b9 2024-11-11T22:17:37,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/559160a6df0f49edaac2c1792c775a16 is 50, key is test_row_0/B:col10/1731363457202/Put/seqid=0 2024-11-11T22:17:37,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:37,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:37,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,447 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into 3474dc378d7e46b6b8102377b6d4c6b9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:37,447 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:37,447 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=13, startTime=1731363457279; duration=0sec 2024-11-11T22:17:37,447 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:37,447 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:37,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742075_1251 (size=12151) 2024-11-11T22:17:37,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/559160a6df0f49edaac2c1792c775a16 2024-11-11T22:17:37,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363517471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:37,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/5bdcaf87d4ef4b85b14b5faeb4916a84 is 50, key is test_row_0/C:col10/1731363457202/Put/seqid=0 2024-11-11T22:17:37,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742076_1252 (size=12151) 2024-11-11T22:17:37,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/5bdcaf87d4ef4b85b14b5faeb4916a84 2024-11-11T22:17:37,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/1716672cce5c41f3a1ae624778a28180 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1716672cce5c41f3a1ae624778a28180 2024-11-11T22:17:37,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1716672cce5c41f3a1ae624778a28180, entries=200, sequenceid=144, filesize=14.2 K 2024-11-11T22:17:37,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/559160a6df0f49edaac2c1792c775a16 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/559160a6df0f49edaac2c1792c775a16 2024-11-11T22:17:37,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/559160a6df0f49edaac2c1792c775a16, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T22:17:37,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/5bdcaf87d4ef4b85b14b5faeb4916a84 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5bdcaf87d4ef4b85b14b5faeb4916a84 2024-11-11T22:17:37,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5bdcaf87d4ef4b85b14b5faeb4916a84, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T22:17:37,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aea091a0b91074e84f623acef361ebf8 in 232ms, sequenceid=144, compaction requested=false 2024-11-11T22:17:37,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:37,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:17:37,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:17:37,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ccfa3c76fb4a48b383d2c658079d63c8 is 50, key is test_row_0/A:col10/1731363457470/Put/seqid=0 2024-11-11T22:17:37,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742077_1253 (size=14541) 2024-11-11T22:17:37,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ccfa3c76fb4a48b383d2c658079d63c8 2024-11-11T22:17:37,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/2d19b42c4ca34b33a5cef21b2a85542c is 50, key is test_row_0/B:col10/1731363457470/Put/seqid=0 2024-11-11T22:17:37,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363517685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:37,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:37,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742078_1254 (size=12151) 2024-11-11T22:17:37,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,787 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/23e16f1d817544328f11703f3d2ea162 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/23e16f1d817544328f11703f3d2ea162 2024-11-11T22:17:37,797 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into 23e16f1d817544328f11703f3d2ea162(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:37,797 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:37,797 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=13, startTime=1731363457278; duration=0sec 2024-11-11T22:17:37,798 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:37,798 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:37,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363517800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:37,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:37,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:17:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:37,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363518010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:38,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/2d19b42c4ca34b33a5cef21b2a85542c 2024-11-11T22:17:38,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ed26976aec9144ca84dd47d035b92031 is 50, key is test_row_0/C:col10/1731363457470/Put/seqid=0 2024-11-11T22:17:38,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363518150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:38,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363518150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:38,156 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_0' in aea091a0b91074e84f623acef361ebf8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:38,156 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in aea091a0b91074e84f623acef361ebf8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:38,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363518150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:38,157 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in aea091a0b91074e84f623acef361ebf8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363518171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:38,180 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_1' in aea091a0b91074e84f623acef361ebf8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:17:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:38,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742079_1255 (size=12151) 2024-11-11T22:17:38,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ed26976aec9144ca84dd47d035b92031 2024-11-11T22:17:38,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ccfa3c76fb4a48b383d2c658079d63c8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ccfa3c76fb4a48b383d2c658079d63c8 2024-11-11T22:17:38,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ccfa3c76fb4a48b383d2c658079d63c8, entries=200, sequenceid=166, filesize=14.2 K 2024-11-11T22:17:38,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/2d19b42c4ca34b33a5cef21b2a85542c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/2d19b42c4ca34b33a5cef21b2a85542c 2024-11-11T22:17:38,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/2d19b42c4ca34b33a5cef21b2a85542c, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T22:17:38,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ed26976aec9144ca84dd47d035b92031 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ed26976aec9144ca84dd47d035b92031 2024-11-11T22:17:38,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ed26976aec9144ca84dd47d035b92031, entries=150, sequenceid=166, filesize=11.9 K 2024-11-11T22:17:38,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for aea091a0b91074e84f623acef361ebf8 in 693ms, sequenceid=166, compaction requested=true 2024-11-11T22:17:38,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:38,290 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:38,290 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:38,291 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:38,291 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:38,291 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,291 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/23e16f1d817544328f11703f3d2ea162, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/559160a6df0f49edaac2c1792c775a16, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/2d19b42c4ca34b33a5cef21b2a85542c] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=35.7 K 2024-11-11T22:17:38,292 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41373 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:38,292 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e16f1d817544328f11703f3d2ea162, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731363457140 2024-11-11T22:17:38,292 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:38,292 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,292 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cc5ccdcee13946eeb6db29586d79f823, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1716672cce5c41f3a1ae624778a28180, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ccfa3c76fb4a48b383d2c658079d63c8] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=40.4 K 2024-11-11T22:17:38,292 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 559160a6df0f49edaac2c1792c775a16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731363457202 2024-11-11T22:17:38,293 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5ccdcee13946eeb6db29586d79f823, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731363457140 2024-11-11T22:17:38,293 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d19b42c4ca34b33a5cef21b2a85542c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731363457463 2024-11-11T22:17:38,297 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1716672cce5c41f3a1ae624778a28180, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731363457197 2024-11-11T22:17:38,297 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccfa3c76fb4a48b383d2c658079d63c8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731363457448 2024-11-11T22:17:38,309 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:38,310 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e61e2c94c3a242b7aa3a0736f25f24aa is 50, key is test_row_0/B:col10/1731363457470/Put/seqid=0 2024-11-11T22:17:38,314 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:38,315 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/54998f0f3e1b4aa89a56c9fd438cae89 is 50, key is test_row_0/A:col10/1731363457470/Put/seqid=0 2024-11-11T22:17:38,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:17:38,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:38,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:38,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742080_1256 (size=12493) 2024-11-11T22:17:38,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ca674c7f7a5a43faaabfd1496ccb7cf8 is 50, key is test_row_0/A:col10/1731363458329/Put/seqid=0 2024-11-11T22:17:38,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742081_1257 (size=12493) 2024-11-11T22:17:38,392 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e61e2c94c3a242b7aa3a0736f25f24aa as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e61e2c94c3a242b7aa3a0736f25f24aa 2024-11-11T22:17:38,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,402 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into e61e2c94c3a242b7aa3a0736f25f24aa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:38,402 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:38,402 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=13, startTime=1731363458290; duration=0sec 2024-11-11T22:17:38,403 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:38,403 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:38,403 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:38,403 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/54998f0f3e1b4aa89a56c9fd438cae89 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/54998f0f3e1b4aa89a56c9fd438cae89 2024-11-11T22:17:38,405 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:38,405 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:38,405 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,405 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3474dc378d7e46b6b8102377b6d4c6b9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5bdcaf87d4ef4b85b14b5faeb4916a84, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ed26976aec9144ca84dd47d035b92031] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=35.7 K 2024-11-11T22:17:38,406 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3474dc378d7e46b6b8102377b6d4c6b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731363457140 2024-11-11T22:17:38,407 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bdcaf87d4ef4b85b14b5faeb4916a84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731363457202 2024-11-11T22:17:38,408 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ed26976aec9144ca84dd47d035b92031, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731363457463 2024-11-11T22:17:38,413 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into 54998f0f3e1b4aa89a56c9fd438cae89(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:38,413 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:38,413 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=13, startTime=1731363458289; duration=0sec 2024-11-11T22:17:38,414 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:38,414 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:17:38,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742082_1258 (size=14541) 2024-11-11T22:17:38,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ca674c7f7a5a43faaabfd1496ccb7cf8 2024-11-11T22:17:38,439 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#216 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:38,440 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/eb5d96a991ba4d659099750344ab1e00 is 50, key is test_row_0/C:col10/1731363457470/Put/seqid=0 2024-11-11T22:17:38,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/840bb8a7f8914a9f96d8c9a9481bff63 is 50, key is test_row_0/B:col10/1731363458329/Put/seqid=0 2024-11-11T22:17:38,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742083_1259 (size=12493) 2024-11-11T22:17:38,506 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/eb5d96a991ba4d659099750344ab1e00 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/eb5d96a991ba4d659099750344ab1e00 2024-11-11T22:17:38,513 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into eb5d96a991ba4d659099750344ab1e00(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:38,513 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:38,513 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=13, startTime=1731363458290; duration=0sec 2024-11-11T22:17:38,513 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:38,513 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:38,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742084_1260 (size=12151) 2024-11-11T22:17:38,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/840bb8a7f8914a9f96d8c9a9481bff63 2024-11-11T22:17:38,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/5dee4717845547ad929837d5572a7070 is 50, key is test_row_0/C:col10/1731363458329/Put/seqid=0 2024-11-11T22:17:38,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363518541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:38,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:38,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:38,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742085_1261 (size=12151) 2024-11-11T22:17:38,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/5dee4717845547ad929837d5572a7070 2024-11-11T22:17:38,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ca674c7f7a5a43faaabfd1496ccb7cf8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ca674c7f7a5a43faaabfd1496ccb7cf8 2024-11-11T22:17:38,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ca674c7f7a5a43faaabfd1496ccb7cf8, entries=200, sequenceid=180, filesize=14.2 K 2024-11-11T22:17:38,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/840bb8a7f8914a9f96d8c9a9481bff63 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/840bb8a7f8914a9f96d8c9a9481bff63 2024-11-11T22:17:38,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/840bb8a7f8914a9f96d8c9a9481bff63, entries=150, sequenceid=180, filesize=11.9 K 2024-11-11T22:17:38,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/5dee4717845547ad929837d5572a7070 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5dee4717845547ad929837d5572a7070 2024-11-11T22:17:38,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5dee4717845547ad929837d5572a7070, entries=150, sequenceid=180, filesize=11.9 K 2024-11-11T22:17:38,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aea091a0b91074e84f623acef361ebf8 in 284ms, sequenceid=180, compaction requested=false 2024-11-11T22:17:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:38,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:17:38,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:38,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:38,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:38,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:38,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:38,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:38,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/cb7cf42143ce4ea6b3761a05e19cbeb3 is 50, key is test_row_0/A:col10/1731363458662/Put/seqid=0 2024-11-11T22:17:38,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:38,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742086_1262 (size=14541) 2024-11-11T22:17:38,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:38,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363518736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:38,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:38,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:38,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:38,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:38,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:38,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363518861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:39,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:39,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:39,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:39,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363519070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:39,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/cb7cf42143ce4ea6b3761a05e19cbeb3 2024-11-11T22:17:39,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7e5c65055ec94666bc769be20d68c643 is 50, key is test_row_0/B:col10/1731363458662/Put/seqid=0 2024-11-11T22:17:39,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:39,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:39,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:39,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:39,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:39,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:39,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742087_1263 (size=12151) 2024-11-11T22:17:39,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7e5c65055ec94666bc769be20d68c643 2024-11-11T22:17:39,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/80d8cbf875db422089cf320a146ace58 is 50, key is test_row_0/C:col10/1731363458662/Put/seqid=0 2024-11-11T22:17:39,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742088_1264 (size=12151) 2024-11-11T22:17:39,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/80d8cbf875db422089cf320a146ace58 2024-11-11T22:17:39,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/cb7cf42143ce4ea6b3761a05e19cbeb3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cb7cf42143ce4ea6b3761a05e19cbeb3 2024-11-11T22:17:39,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cb7cf42143ce4ea6b3761a05e19cbeb3, entries=200, sequenceid=203, filesize=14.2 K 2024-11-11T22:17:39,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7e5c65055ec94666bc769be20d68c643 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7e5c65055ec94666bc769be20d68c643 2024-11-11T22:17:39,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7e5c65055ec94666bc769be20d68c643, entries=150, sequenceid=203, filesize=11.9 K 2024-11-11T22:17:39,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/80d8cbf875db422089cf320a146ace58 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/80d8cbf875db422089cf320a146ace58 2024-11-11T22:17:39,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/80d8cbf875db422089cf320a146ace58, entries=150, sequenceid=203, filesize=11.9 K 2024-11-11T22:17:39,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for aea091a0b91074e84f623acef361ebf8 in 636ms, sequenceid=203, compaction requested=true 2024-11-11T22:17:39,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:39,301 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:39,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:39,304 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:39,304 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:39,304 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:39,304 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/54998f0f3e1b4aa89a56c9fd438cae89, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ca674c7f7a5a43faaabfd1496ccb7cf8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cb7cf42143ce4ea6b3761a05e19cbeb3] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=40.6 K 2024-11-11T22:17:39,305 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54998f0f3e1b4aa89a56c9fd438cae89, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731363457463 2024-11-11T22:17:39,306 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca674c7f7a5a43faaabfd1496ccb7cf8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731363457664 2024-11-11T22:17:39,307 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb7cf42143ce4ea6b3761a05e19cbeb3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731363458427 2024-11-11T22:17:39,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:39,309 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:39,311 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:39,311 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:39,311 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:39,311 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e61e2c94c3a242b7aa3a0736f25f24aa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/840bb8a7f8914a9f96d8c9a9481bff63, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7e5c65055ec94666bc769be20d68c643] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=35.9 K 2024-11-11T22:17:39,313 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e61e2c94c3a242b7aa3a0736f25f24aa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731363457463 2024-11-11T22:17:39,314 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 840bb8a7f8914a9f96d8c9a9481bff63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731363457672 2024-11-11T22:17:39,315 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5c65055ec94666bc769be20d68c643, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731363458427 2024-11-11T22:17:39,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:39,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:39,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:39,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:39,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T22:17:39,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:39,328 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:17:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:39,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:39,339 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:39,339 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ab477fb548534705a337f922413f695f is 50, key is test_row_0/A:col10/1731363458662/Put/seqid=0 2024-11-11T22:17:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/fe22e98f4c4d481cb3fb8ff9ab5cf817 is 50, key is test_row_0/A:col10/1731363458733/Put/seqid=0 2024-11-11T22:17:39,359 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:39,359 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/404978326745416d93d4d44ff85b4d22 is 50, key is test_row_0/B:col10/1731363458662/Put/seqid=0 2024-11-11T22:17:39,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:39,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742089_1265 (size=12595) 2024-11-11T22:17:39,402 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ab477fb548534705a337f922413f695f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ab477fb548534705a337f922413f695f 2024-11-11T22:17:39,411 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into ab477fb548534705a337f922413f695f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:39,411 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:39,411 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=13, startTime=1731363459301; duration=0sec 2024-11-11T22:17:39,411 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:39,412 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:39,412 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:39,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742090_1266 (size=12151) 2024-11-11T22:17:39,416 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:39,416 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:39,416 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:39,416 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/eb5d96a991ba4d659099750344ab1e00, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5dee4717845547ad929837d5572a7070, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/80d8cbf875db422089cf320a146ace58] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=35.9 K 2024-11-11T22:17:39,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742091_1267 (size=12595) 2024-11-11T22:17:39,417 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/fe22e98f4c4d481cb3fb8ff9ab5cf817 2024-11-11T22:17:39,417 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb5d96a991ba4d659099750344ab1e00, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731363457463 2024-11-11T22:17:39,417 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dee4717845547ad929837d5572a7070, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731363457672 2024-11-11T22:17:39,419 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80d8cbf875db422089cf320a146ace58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731363458427 2024-11-11T22:17:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:17:39,448 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/404978326745416d93d4d44ff85b4d22 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/404978326745416d93d4d44ff85b4d22 2024-11-11T22:17:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7a13d1af7e2e42399f6c934cb9a1c71b is 50, key is test_row_0/B:col10/1731363458733/Put/seqid=0 2024-11-11T22:17:39,458 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into 404978326745416d93d4d44ff85b4d22(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:39,458 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:39,458 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=13, startTime=1731363459308; duration=0sec 2024-11-11T22:17:39,458 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:39,458 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:39,465 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:39,466 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/d20658a2e79b49d99d5acac20ea91e48 is 50, key is test_row_0/C:col10/1731363458662/Put/seqid=0 2024-11-11T22:17:39,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742092_1268 (size=12151) 2024-11-11T22:17:39,512 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7a13d1af7e2e42399f6c934cb9a1c71b 2024-11-11T22:17:39,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742093_1269 (size=12595) 2024-11-11T22:17:39,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/f89e31bad0134fbab7230d9fa3da8da3 is 50, key is test_row_0/C:col10/1731363458733/Put/seqid=0 2024-11-11T22:17:39,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363519531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:39,534 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/d20658a2e79b49d99d5acac20ea91e48 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d20658a2e79b49d99d5acac20ea91e48 2024-11-11T22:17:39,554 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into d20658a2e79b49d99d5acac20ea91e48(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:39,554 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:39,554 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=13, startTime=1731363459322; duration=0sec 2024-11-11T22:17:39,554 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:39,554 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:39,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742094_1270 (size=12151) 2024-11-11T22:17:39,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363519639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:39,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363519849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:39,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:39,978 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/f89e31bad0134fbab7230d9fa3da8da3 2024-11-11T22:17:39,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/fe22e98f4c4d481cb3fb8ff9ab5cf817 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/fe22e98f4c4d481cb3fb8ff9ab5cf817 2024-11-11T22:17:39,992 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/fe22e98f4c4d481cb3fb8ff9ab5cf817, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T22:17:39,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/7a13d1af7e2e42399f6c934cb9a1c71b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7a13d1af7e2e42399f6c934cb9a1c71b 2024-11-11T22:17:39,999 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7a13d1af7e2e42399f6c934cb9a1c71b, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T22:17:40,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/f89e31bad0134fbab7230d9fa3da8da3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f89e31bad0134fbab7230d9fa3da8da3 2024-11-11T22:17:40,012 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f89e31bad0134fbab7230d9fa3da8da3, entries=150, sequenceid=215, filesize=11.9 K 2024-11-11T22:17:40,021 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for aea091a0b91074e84f623acef361ebf8 in 694ms, sequenceid=215, compaction requested=false 2024-11-11T22:17:40,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:40,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:40,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T22:17:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T22:17:40,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T22:17:40,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7380 sec 2024-11-11T22:17:40,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.7510 sec 2024-11-11T22:17:40,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363459470, max timestamp=1731363460166Num uniques 859; 2024-11-11T22:17:40,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:40,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-11T22:17:40,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:40,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:40,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:40,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:40,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363459470, max timestamp=1731363460166Num uniques 859; 2024-11-11T22:17:40,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:40,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:40,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:40,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/48522a08b46941409ddc2adfe7fbbd2b is 50, key is test_row_0/A:col10/1731363459504/Put/seqid=0 2024-11-11T22:17:40,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742095_1271 (size=12151) 2024-11-11T22:17:40,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/48522a08b46941409ddc2adfe7fbbd2b 2024-11-11T22:17:40,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/40cbcce9b61f4d9f9a5e8047f8a0607c is 50, key is test_row_0/B:col10/1731363459504/Put/seqid=0 2024-11-11T22:17:40,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742096_1272 (size=12151) 2024-11-11T22:17:40,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/40cbcce9b61f4d9f9a5e8047f8a0607c 2024-11-11T22:17:40,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/65e7e3d2340848b88d242652fdf25197 is 50, key is test_row_0/C:col10/1731363459504/Put/seqid=0 2024-11-11T22:17:40,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363460308, max timestamp=1731363460346Num uniques 859; 2024-11-11T22:17:40,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:40,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363460308, max timestamp=1731363460346Num uniques 859; 2024-11-11T22:17:40,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:40,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363460308, max timestamp=1731363460346Num uniques 859; 2024-11-11T22:17:40,353 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:17:40,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742097_1273 (size=12151) 2024-11-11T22:17:40,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363520502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:40,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:40,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363520620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:40,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/65e7e3d2340848b88d242652fdf25197 2024-11-11T22:17:40,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/48522a08b46941409ddc2adfe7fbbd2b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/48522a08b46941409ddc2adfe7fbbd2b 2024-11-11T22:17:40,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/48522a08b46941409ddc2adfe7fbbd2b, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T22:17:40,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/40cbcce9b61f4d9f9a5e8047f8a0607c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/40cbcce9b61f4d9f9a5e8047f8a0607c 2024-11-11T22:17:40,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/40cbcce9b61f4d9f9a5e8047f8a0607c, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T22:17:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/65e7e3d2340848b88d242652fdf25197 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/65e7e3d2340848b88d242652fdf25197 2024-11-11T22:17:40,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/65e7e3d2340848b88d242652fdf25197, entries=150, sequenceid=239, filesize=11.9 K 2024-11-11T22:17:40,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for aea091a0b91074e84f623acef361ebf8 in 671ms, sequenceid=239, compaction requested=true 2024-11-11T22:17:40,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:40,842 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:40,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:40,843 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:40,844 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:40,844 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:40,847 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:40,847 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ab477fb548534705a337f922413f695f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/fe22e98f4c4d481cb3fb8ff9ab5cf817, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/48522a08b46941409ddc2adfe7fbbd2b] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=36.0 K 2024-11-11T22:17:40,848 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:40,848 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:40,848 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:40,849 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/404978326745416d93d4d44ff85b4d22, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7a13d1af7e2e42399f6c934cb9a1c71b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/40cbcce9b61f4d9f9a5e8047f8a0607c] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=36.0 K 2024-11-11T22:17:40,849 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab477fb548534705a337f922413f695f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731363458427 2024-11-11T22:17:40,850 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 404978326745416d93d4d44ff85b4d22, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731363458427 2024-11-11T22:17:40,850 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe22e98f4c4d481cb3fb8ff9ab5cf817, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731363458695 2024-11-11T22:17:40,850 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a13d1af7e2e42399f6c934cb9a1c71b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731363458695 2024-11-11T22:17:40,850 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48522a08b46941409ddc2adfe7fbbd2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731363459470 2024-11-11T22:17:40,851 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 40cbcce9b61f4d9f9a5e8047f8a0607c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731363459470 2024-11-11T22:17:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:40,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:40,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:40,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:40,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:40,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T22:17:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:40,874 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:40,875 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/34fd705fb139489d8574d86beb89f399 is 50, key is test_row_0/B:col10/1731363459504/Put/seqid=0 2024-11-11T22:17:40,886 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:40,886 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/2a3286d106ad447eabb83501048bfe66 is 50, key is test_row_0/A:col10/1731363459504/Put/seqid=0 2024-11-11T22:17:40,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/11dd7054c508410eb3a4f9758bb23a3e is 50, key is test_row_0/A:col10/1731363460494/Put/seqid=0 2024-11-11T22:17:40,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742098_1274 (size=12697) 2024-11-11T22:17:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363520945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:40,957 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/34fd705fb139489d8574d86beb89f399 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/34fd705fb139489d8574d86beb89f399 2024-11-11T22:17:40,967 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into 34fd705fb139489d8574d86beb89f399(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:40,967 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:40,967 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=13, startTime=1731363460843; duration=0sec 2024-11-11T22:17:40,967 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:40,967 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:40,967 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:40,969 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:40,969 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:40,969 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:40,970 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d20658a2e79b49d99d5acac20ea91e48, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f89e31bad0134fbab7230d9fa3da8da3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/65e7e3d2340848b88d242652fdf25197] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=36.0 K 2024-11-11T22:17:40,971 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d20658a2e79b49d99d5acac20ea91e48, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731363458427 2024-11-11T22:17:40,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742099_1275 (size=12697) 2024-11-11T22:17:40,971 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f89e31bad0134fbab7230d9fa3da8da3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731363458695 2024-11-11T22:17:40,972 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e7e3d2340848b88d242652fdf25197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731363459470 2024-11-11T22:17:41,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742100_1276 (size=14741) 2024-11-11T22:17:41,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/11dd7054c508410eb3a4f9758bb23a3e 2024-11-11T22:17:41,021 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:41,022 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/996ba58ec9b74a8ea37d7dce6ae6eb0b is 50, key is test_row_0/C:col10/1731363459504/Put/seqid=0 2024-11-11T22:17:41,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/c59fedabc26249b49140a9d32619d23d is 50, key is test_row_0/B:col10/1731363460494/Put/seqid=0 2024-11-11T22:17:41,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742101_1277 (size=12697) 2024-11-11T22:17:41,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363521060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:41,076 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/996ba58ec9b74a8ea37d7dce6ae6eb0b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/996ba58ec9b74a8ea37d7dce6ae6eb0b 2024-11-11T22:17:41,084 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into 996ba58ec9b74a8ea37d7dce6ae6eb0b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:41,084 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:41,084 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=13, startTime=1731363460862; duration=0sec 2024-11-11T22:17:41,084 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:41,085 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:41,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742102_1278 (size=12301) 2024-11-11T22:17:41,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/c59fedabc26249b49140a9d32619d23d 2024-11-11T22:17:41,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/909c34022619415897f06842e0bc37b9 is 50, key is test_row_0/C:col10/1731363460494/Put/seqid=0 2024-11-11T22:17:41,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742103_1279 (size=12301) 2024-11-11T22:17:41,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363521279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:41,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:41,436 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/2a3286d106ad447eabb83501048bfe66 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/2a3286d106ad447eabb83501048bfe66 2024-11-11T22:17:41,444 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into 2a3286d106ad447eabb83501048bfe66(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:41,444 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:41,444 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=13, startTime=1731363460842; duration=0sec 2024-11-11T22:17:41,445 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:41,445 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T22:17:41,457 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:41,458 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T22:17:41,462 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:17:41,463 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:41,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:17:41,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/909c34022619415897f06842e0bc37b9 2024-11-11T22:17:41,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/11dd7054c508410eb3a4f9758bb23a3e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11dd7054c508410eb3a4f9758bb23a3e 2024-11-11T22:17:41,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11dd7054c508410eb3a4f9758bb23a3e, entries=200, sequenceid=276, filesize=14.4 K 2024-11-11T22:17:41,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363521589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:41,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/c59fedabc26249b49140a9d32619d23d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/c59fedabc26249b49140a9d32619d23d 2024-11-11T22:17:41,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/c59fedabc26249b49140a9d32619d23d, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T22:17:41,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/909c34022619415897f06842e0bc37b9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/909c34022619415897f06842e0bc37b9 2024-11-11T22:17:41,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/909c34022619415897f06842e0bc37b9, entries=150, sequenceid=276, filesize=12.0 K 2024-11-11T22:17:41,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for aea091a0b91074e84f623acef361ebf8 in 749ms, sequenceid=276, compaction requested=false 2024-11-11T22:17:41,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:41,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T22:17:41,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:41,617 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:17:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:41,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/620144088bb6417886a72254b230a4ae is 50, key is test_row_1/A:col10/1731363460942/Put/seqid=0 2024-11-11T22:17:41,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742104_1280 (size=9857) 2024-11-11T22:17:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:17:42,076 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/620144088bb6417886a72254b230a4ae 2024-11-11T22:17:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:17:42,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/f91b6a06bc624020b98fffee8984d629 is 50, key is test_row_1/B:col10/1731363460942/Put/seqid=0 2024-11-11T22:17:42,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742105_1281 (size=9857) 2024-11-11T22:17:42,146 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/f91b6a06bc624020b98fffee8984d629 2024-11-11T22:17:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc is 50, key is test_row_1/C:col10/1731363460942/Put/seqid=0 2024-11-11T22:17:42,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742106_1282 (size=9857) 2024-11-11T22:17:42,172 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc 2024-11-11T22:17:42,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/620144088bb6417886a72254b230a4ae as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/620144088bb6417886a72254b230a4ae 2024-11-11T22:17:42,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363462148, max timestamp=1731363462176Num uniques 859; 2024-11-11T22:17:42,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363462148, max timestamp=1731363462176Num uniques 859; 2024-11-11T22:17:42,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:42,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:42,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363462148, max timestamp=1731363462176Num uniques 859; 2024-11-11T22:17:42,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:17:42,194 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/620144088bb6417886a72254b230a4ae, entries=100, sequenceid=289, filesize=9.6 K 2024-11-11T22:17:42,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/f91b6a06bc624020b98fffee8984d629 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/f91b6a06bc624020b98fffee8984d629 2024-11-11T22:17:42,202 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/f91b6a06bc624020b98fffee8984d629, entries=100, sequenceid=289, filesize=9.6 K 2024-11-11T22:17:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc 2024-11-11T22:17:42,210 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc, entries=100, sequenceid=289, filesize=9.6 K 2024-11-11T22:17:42,211 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=107.34 KB/109920 for aea091a0b91074e84f623acef361ebf8 in 594ms, sequenceid=289, compaction requested=true 2024-11-11T22:17:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T22:17:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T22:17:42,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=114.05 KB heapSize=320.53 KB 2024-11-11T22:17:42,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:42,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:42,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:42,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:42,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:42,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:42,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T22:17:42,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 751 msec 2024-11-11T22:17:42,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 767 msec 2024-11-11T22:17:42,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/b11087115b31437cb88e0fd2a7b457ac is 50, key is test_row_0/A:col10/1731363462199/Put/seqid=0 2024-11-11T22:17:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363522243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363522245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363522246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363522247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363522251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742107_1283 (size=22065) 2024-11-11T22:17:42,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/b11087115b31437cb88e0fd2a7b457ac 2024-11-11T22:17:42,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/ad2474b5a8f343f880803d0998d9f149 is 50, key is test_row_0/B:col10/1731363462199/Put/seqid=0 2024-11-11T22:17:42,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742108_1284 (size=12301) 2024-11-11T22:17:42,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363522359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363522359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363522370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363522370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363522370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363522569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363522580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363522580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363522580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363522580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T22:17:42,599 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:42,609 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T22:17:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:17:42,618 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:42,619 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:42,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:17:42,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/ad2474b5a8f343f880803d0998d9f149 2024-11-11T22:17:42,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/993e9b4b40254d738fe4dfc0687cf6a4 is 50, key is test_row_0/C:col10/1731363462199/Put/seqid=0 2024-11-11T22:17:42,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742109_1285 (size=12301) 2024-11-11T22:17:42,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/993e9b4b40254d738fe4dfc0687cf6a4 2024-11-11T22:17:42,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/b11087115b31437cb88e0fd2a7b457ac as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b11087115b31437cb88e0fd2a7b457ac 2024-11-11T22:17:42,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b11087115b31437cb88e0fd2a7b457ac, entries=350, sequenceid=322, filesize=21.5 K 2024-11-11T22:17:42,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/ad2474b5a8f343f880803d0998d9f149 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/ad2474b5a8f343f880803d0998d9f149 2024-11-11T22:17:42,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/ad2474b5a8f343f880803d0998d9f149, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T22:17:42,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/993e9b4b40254d738fe4dfc0687cf6a4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/993e9b4b40254d738fe4dfc0687cf6a4 2024-11-11T22:17:42,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:17:42,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:42,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:42,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:42,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:42,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:42,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/993e9b4b40254d738fe4dfc0687cf6a4, entries=150, sequenceid=322, filesize=12.0 K 2024-11-11T22:17:42,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~320.48 KB/328176, currentSize=67.09 KB/68700 for aea091a0b91074e84f623acef361ebf8 in 564ms, sequenceid=322, compaction requested=true 2024-11-11T22:17:42,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:42,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:42,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:42,780 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:42,780 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:42,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:42,782 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:42,782 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47156 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:42,782 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:42,782 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:42,782 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:42,782 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:42,782 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/34fd705fb139489d8574d86beb89f399, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/c59fedabc26249b49140a9d32619d23d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/f91b6a06bc624020b98fffee8984d629, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/ad2474b5a8f343f880803d0998d9f149] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=46.1 K 2024-11-11T22:17:42,782 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/2a3286d106ad447eabb83501048bfe66, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11dd7054c508410eb3a4f9758bb23a3e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/620144088bb6417886a72254b230a4ae, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b11087115b31437cb88e0fd2a7b457ac] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=58.0 K 2024-11-11T22:17:42,783 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 34fd705fb139489d8574d86beb89f399, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731363459470 2024-11-11T22:17:42,783 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a3286d106ad447eabb83501048bfe66, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731363459470 2024-11-11T22:17:42,783 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c59fedabc26249b49140a9d32619d23d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731363460449 2024-11-11T22:17:42,783 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11dd7054c508410eb3a4f9758bb23a3e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731363460449 2024-11-11T22:17:42,784 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f91b6a06bc624020b98fffee8984d629, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731363460926 2024-11-11T22:17:42,784 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 620144088bb6417886a72254b230a4ae, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731363460926 2024-11-11T22:17:42,784 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2474b5a8f343f880803d0998d9f149, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731363462181 2024-11-11T22:17:42,784 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b11087115b31437cb88e0fd2a7b457ac, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731363462181 2024-11-11T22:17:42,802 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:42,802 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/3ebeb045cf134e52950671919b46f8e7 is 50, key is test_row_0/B:col10/1731363462199/Put/seqid=0 2024-11-11T22:17:42,806 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#244 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:42,806 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/47780e288dbd4a6191342bf753da0a96 is 50, key is test_row_0/A:col10/1731363462199/Put/seqid=0 2024-11-11T22:17:42,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742110_1286 (size=12983) 2024-11-11T22:17:42,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742111_1287 (size=12983) 2024-11-11T22:17:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:42,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T22:17:42,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:42,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:42,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:42,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:42,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:42,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:42,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/67958ac9d0ff49b7a8170de3bc08d9b3 is 50, key is test_row_0/A:col10/1731363462245/Put/seqid=0 2024-11-11T22:17:42,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742112_1288 (size=14741) 2024-11-11T22:17:42,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/67958ac9d0ff49b7a8170de3bc08d9b3 2024-11-11T22:17:42,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/0bf394a86c084ea3981eb640587520ba is 50, key is test_row_0/B:col10/1731363462245/Put/seqid=0 2024-11-11T22:17:42,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742113_1289 (size=12301) 2024-11-11T22:17:42,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:17:42,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:42,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:42,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:42,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:42,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:17:42,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363522937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363522933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363522940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363522941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363522942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:42,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363523049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363523049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363523049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363523060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363523060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:17:43,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:43,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,223 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/3ebeb045cf134e52950671919b46f8e7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3ebeb045cf134e52950671919b46f8e7 2024-11-11T22:17:43,228 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/47780e288dbd4a6191342bf753da0a96 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/47780e288dbd4a6191342bf753da0a96 2024-11-11T22:17:43,230 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into 3ebeb045cf134e52950671919b46f8e7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:43,230 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:43,230 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=12, startTime=1731363462780; duration=0sec 2024-11-11T22:17:43,230 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:43,230 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:43,230 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:43,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:17:43,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:43,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,238 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47156 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:43,239 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:43,239 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,239 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/996ba58ec9b74a8ea37d7dce6ae6eb0b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/909c34022619415897f06842e0bc37b9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/993e9b4b40254d738fe4dfc0687cf6a4] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=46.1 K 2024-11-11T22:17:43,239 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 996ba58ec9b74a8ea37d7dce6ae6eb0b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731363459470 2024-11-11T22:17:43,247 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 909c34022619415897f06842e0bc37b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731363460449 2024-11-11T22:17:43,248 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f9d0ebc55dd54fdb8ccaac32ffd48bfc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731363460926 2024-11-11T22:17:43,248 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 993e9b4b40254d738fe4dfc0687cf6a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731363462181 2024-11-11T22:17:43,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:17:43,252 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into 47780e288dbd4a6191342bf753da0a96(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:43,252 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:43,252 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=12, startTime=1731363462780; duration=0sec 2024-11-11T22:17:43,252 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:43,252 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:43,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363523260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363523260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363523260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,271 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:43,272 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/286d7b247c264be58661d9d034966c1c is 50, key is test_row_0/C:col10/1731363462199/Put/seqid=0 2024-11-11T22:17:43,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363523269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363523269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/0bf394a86c084ea3981eb640587520ba 2024-11-11T22:17:43,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742114_1290 (size=12983) 2024-11-11T22:17:43,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c685facbcb0647bd867e9b62a7d0329b is 50, key is test_row_0/C:col10/1731363462245/Put/seqid=0 2024-11-11T22:17:43,334 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/286d7b247c264be58661d9d034966c1c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/286d7b247c264be58661d9d034966c1c 2024-11-11T22:17:43,340 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into 286d7b247c264be58661d9d034966c1c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:43,340 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:43,340 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=12, startTime=1731363462781; duration=0sec 2024-11-11T22:17:43,340 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:43,340 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:43,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742115_1291 (size=12301) 2024-11-11T22:17:43,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:17:43,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:43,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:17:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363523570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363523570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363523570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363523585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363523585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:43,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:43,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:17:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:43,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c685facbcb0647bd867e9b62a7d0329b 2024-11-11T22:17:43,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/67958ac9d0ff49b7a8170de3bc08d9b3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/67958ac9d0ff49b7a8170de3bc08d9b3 2024-11-11T22:17:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:17:43,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/67958ac9d0ff49b7a8170de3bc08d9b3, entries=200, sequenceid=336, filesize=14.4 K 2024-11-11T22:17:43,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/0bf394a86c084ea3981eb640587520ba as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0bf394a86c084ea3981eb640587520ba 2024-11-11T22:17:43,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0bf394a86c084ea3981eb640587520ba, entries=150, sequenceid=336, filesize=12.0 K 2024-11-11T22:17:43,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c685facbcb0647bd867e9b62a7d0329b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c685facbcb0647bd867e9b62a7d0329b 2024-11-11T22:17:43,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c685facbcb0647bd867e9b62a7d0329b, entries=150, sequenceid=336, filesize=12.0 K 2024-11-11T22:17:43,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for aea091a0b91074e84f623acef361ebf8 in 902ms, sequenceid=336, compaction requested=false 2024-11-11T22:17:43,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:43,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T22:17:43,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:43,853 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T22:17:43,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:43,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:43,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:43,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:43,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:43,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/4ccaccdfda8b4866b4c43162bcea464e is 50, key is test_row_0/A:col10/1731363462941/Put/seqid=0 2024-11-11T22:17:43,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742116_1292 (size=12301) 2024-11-11T22:17:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:44,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:44,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363524101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363524104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363524105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363524106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363524106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363524209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363524219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363524220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363524220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363524220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,275 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/4ccaccdfda8b4866b4c43162bcea464e 2024-11-11T22:17:44,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e954567db4a54e1ba51b77d9b14b08ba is 50, key is test_row_0/B:col10/1731363462941/Put/seqid=0 2024-11-11T22:17:44,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742117_1293 (size=12301) 2024-11-11T22:17:44,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363524421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363524431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363524431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363524431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363524431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,698 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e954567db4a54e1ba51b77d9b14b08ba 2024-11-11T22:17:44,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ab385c7b1c234d5eb94ac39b53262a45 is 50, key is test_row_0/C:col10/1731363462941/Put/seqid=0 2024-11-11T22:17:44,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363524729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363524740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363524740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363524740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742118_1294 (size=12301) 2024-11-11T22:17:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363524740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:17:45,145 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ab385c7b1c234d5eb94ac39b53262a45 2024-11-11T22:17:45,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/4ccaccdfda8b4866b4c43162bcea464e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/4ccaccdfda8b4866b4c43162bcea464e 2024-11-11T22:17:45,153 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/4ccaccdfda8b4866b4c43162bcea464e, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T22:17:45,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e954567db4a54e1ba51b77d9b14b08ba as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e954567db4a54e1ba51b77d9b14b08ba 2024-11-11T22:17:45,161 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e954567db4a54e1ba51b77d9b14b08ba, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T22:17:45,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/ab385c7b1c234d5eb94ac39b53262a45 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ab385c7b1c234d5eb94ac39b53262a45 2024-11-11T22:17:45,166 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ab385c7b1c234d5eb94ac39b53262a45, entries=150, sequenceid=357, filesize=12.0 K 2024-11-11T22:17:45,167 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for aea091a0b91074e84f623acef361ebf8 in 1315ms, sequenceid=357, compaction requested=true 2024-11-11T22:17:45,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:45,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:45,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T22:17:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T22:17:45,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T22:17:45,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5500 sec 2024-11-11T22:17:45,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.5590 sec 2024-11-11T22:17:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:45,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T22:17:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:45,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:45,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/066aab1904f34aa9959bcd5a45ba018e is 50, key is test_row_0/A:col10/1731363464102/Put/seqid=0 2024-11-11T22:17:45,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742119_1295 (size=14741) 2024-11-11T22:17:45,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363525268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363525271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363525271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363525272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363525273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363525379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363525390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363525390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363525390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363525390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363525589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363525599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363525599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363525599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363525599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/066aab1904f34aa9959bcd5a45ba018e 2024-11-11T22:17:45,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/4c47da336646434dbfca872b4248ce55 is 50, key is test_row_0/B:col10/1731363464102/Put/seqid=0 2024-11-11T22:17:45,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742120_1296 (size=12301) 2024-11-11T22:17:45,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363525899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363525909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363525910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363525910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363525910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:45,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/4c47da336646434dbfca872b4248ce55 2024-11-11T22:17:46,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/3471a1caf93145d1bea000c8ca5a9950 is 50, key is test_row_0/C:col10/1731363464102/Put/seqid=0 2024-11-11T22:17:46,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742121_1297 (size=12301) 2024-11-11T22:17:46,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/3471a1caf93145d1bea000c8ca5a9950 2024-11-11T22:17:46,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/066aab1904f34aa9959bcd5a45ba018e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/066aab1904f34aa9959bcd5a45ba018e 2024-11-11T22:17:46,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/066aab1904f34aa9959bcd5a45ba018e, entries=200, sequenceid=372, filesize=14.4 K 2024-11-11T22:17:46,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/4c47da336646434dbfca872b4248ce55 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/4c47da336646434dbfca872b4248ce55 2024-11-11T22:17:46,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/4c47da336646434dbfca872b4248ce55, entries=150, sequenceid=372, filesize=12.0 K 2024-11-11T22:17:46,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/3471a1caf93145d1bea000c8ca5a9950 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3471a1caf93145d1bea000c8ca5a9950 2024-11-11T22:17:46,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3471a1caf93145d1bea000c8ca5a9950, entries=150, sequenceid=372, filesize=12.0 K 2024-11-11T22:17:46,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for aea091a0b91074e84f623acef361ebf8 in 868ms, sequenceid=372, compaction requested=true 2024-11-11T22:17:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:46,119 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:46,119 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:46,120 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:46,120 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:46,121 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:46,121 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/47780e288dbd4a6191342bf753da0a96, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/67958ac9d0ff49b7a8170de3bc08d9b3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/4ccaccdfda8b4866b4c43162bcea464e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/066aab1904f34aa9959bcd5a45ba018e] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=53.5 K 2024-11-11T22:17:46,122 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:46,122 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47780e288dbd4a6191342bf753da0a96, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731363462181 2024-11-11T22:17:46,122 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:46,122 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:46,122 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3ebeb045cf134e52950671919b46f8e7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0bf394a86c084ea3981eb640587520ba, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e954567db4a54e1ba51b77d9b14b08ba, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/4c47da336646434dbfca872b4248ce55] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=48.7 K 2024-11-11T22:17:46,123 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67958ac9d0ff49b7a8170de3bc08d9b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731363462240 2024-11-11T22:17:46,123 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ebeb045cf134e52950671919b46f8e7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731363462181 2024-11-11T22:17:46,123 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ccaccdfda8b4866b4c43162bcea464e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731363462931 2024-11-11T22:17:46,123 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf394a86c084ea3981eb640587520ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731363462244 2024-11-11T22:17:46,123 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e954567db4a54e1ba51b77d9b14b08ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731363462931 2024-11-11T22:17:46,123 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 066aab1904f34aa9959bcd5a45ba018e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731363464102 2024-11-11T22:17:46,124 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c47da336646434dbfca872b4248ce55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731363464102 2024-11-11T22:17:46,147 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:46,148 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/273066f6c8d247c5bd4dcd88e338102d is 50, key is test_row_0/B:col10/1731363464102/Put/seqid=0 2024-11-11T22:17:46,151 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:46,152 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/6a7efd98fbc74b2d8e03d6438688a218 is 50, key is test_row_0/A:col10/1731363464102/Put/seqid=0 2024-11-11T22:17:46,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742122_1298 (size=13119) 2024-11-11T22:17:46,168 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/273066f6c8d247c5bd4dcd88e338102d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/273066f6c8d247c5bd4dcd88e338102d 2024-11-11T22:17:46,173 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into 273066f6c8d247c5bd4dcd88e338102d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:46,173 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:46,173 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=12, startTime=1731363466119; duration=0sec 2024-11-11T22:17:46,173 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:46,173 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:46,173 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:46,175 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:46,175 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:46,175 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:46,175 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/286d7b247c264be58661d9d034966c1c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c685facbcb0647bd867e9b62a7d0329b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ab385c7b1c234d5eb94ac39b53262a45, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3471a1caf93145d1bea000c8ca5a9950] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=48.7 K 2024-11-11T22:17:46,177 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 286d7b247c264be58661d9d034966c1c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731363462181 2024-11-11T22:17:46,177 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c685facbcb0647bd867e9b62a7d0329b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731363462244 2024-11-11T22:17:46,177 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ab385c7b1c234d5eb94ac39b53262a45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731363462931 2024-11-11T22:17:46,177 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3471a1caf93145d1bea000c8ca5a9950, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731363464102 2024-11-11T22:17:46,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742123_1299 (size=13119) 2024-11-11T22:17:46,187 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:46,187 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/b015b933bed44dbfa9da8ec4b86048a7 is 50, key is test_row_0/C:col10/1731363464102/Put/seqid=0 2024-11-11T22:17:46,192 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/6a7efd98fbc74b2d8e03d6438688a218 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/6a7efd98fbc74b2d8e03d6438688a218 2024-11-11T22:17:46,199 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into 6a7efd98fbc74b2d8e03d6438688a218(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:46,199 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:46,199 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=12, startTime=1731363466119; duration=0sec 2024-11-11T22:17:46,199 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:46,199 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:46,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742124_1300 (size=13119) 2024-11-11T22:17:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:46,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T22:17:46,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:46,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:46,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:46,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:46,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:46,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:46,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/31de3ceeab344784a8f10b751c010a6c is 50, key is test_row_0/A:col10/1731363465272/Put/seqid=0 2024-11-11T22:17:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742125_1301 (size=14741) 2024-11-11T22:17:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363526428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363526429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363526430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363526431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363526431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363526539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363526540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363526540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363526540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363526543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,606 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/b015b933bed44dbfa9da8ec4b86048a7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/b015b933bed44dbfa9da8ec4b86048a7 2024-11-11T22:17:46,614 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into b015b933bed44dbfa9da8ec4b86048a7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:46,614 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:46,614 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=12, startTime=1731363466119; duration=0sec 2024-11-11T22:17:46,614 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:46,614 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:46,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363526749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363526749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363526749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363526749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363526749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:46,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T22:17:46,779 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:46,779 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T22:17:46,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:17:46,782 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:46,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:46,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/31de3ceeab344784a8f10b751c010a6c 2024-11-11T22:17:46,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e24abe3703d04d81bfd067159f24a6dd is 50, key is test_row_0/B:col10/1731363465272/Put/seqid=0 2024-11-11T22:17:46,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742126_1302 (size=12301) 2024-11-11T22:17:46,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e24abe3703d04d81bfd067159f24a6dd 2024-11-11T22:17:46,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/8784daf0f03d4e0da729d4b02fe208f3 is 50, key is test_row_0/C:col10/1731363465272/Put/seqid=0 2024-11-11T22:17:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:17:46,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742127_1303 (size=12301) 2024-11-11T22:17:46,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/8784daf0f03d4e0da729d4b02fe208f3 2024-11-11T22:17:46,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/31de3ceeab344784a8f10b751c010a6c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/31de3ceeab344784a8f10b751c010a6c 2024-11-11T22:17:46,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/31de3ceeab344784a8f10b751c010a6c, entries=200, sequenceid=392, filesize=14.4 K 2024-11-11T22:17:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e24abe3703d04d81bfd067159f24a6dd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e24abe3703d04d81bfd067159f24a6dd 2024-11-11T22:17:46,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e24abe3703d04d81bfd067159f24a6dd, entries=150, sequenceid=392, filesize=12.0 K 2024-11-11T22:17:46,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/8784daf0f03d4e0da729d4b02fe208f3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8784daf0f03d4e0da729d4b02fe208f3 2024-11-11T22:17:46,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8784daf0f03d4e0da729d4b02fe208f3, entries=150, sequenceid=392, filesize=12.0 K 2024-11-11T22:17:46,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for aea091a0b91074e84f623acef361ebf8 in 509ms, sequenceid=392, compaction requested=false 2024-11-11T22:17:46,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:46,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T22:17:46,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:46,934 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T22:17:46,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:46,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:46,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ae80cc87601446bda92bd415f3644b84 is 50, key is test_row_0/A:col10/1731363466429/Put/seqid=0 2024-11-11T22:17:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742128_1304 (size=12301) 2024-11-11T22:17:46,979 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ae80cc87601446bda92bd415f3644b84 2024-11-11T22:17:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e15e1f532fb540259abbc86f2dea693c is 50, key is test_row_0/B:col10/1731363466429/Put/seqid=0 2024-11-11T22:17:47,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742129_1305 (size=12301) 2024-11-11T22:17:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:47,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:47,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363527072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363527075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363527076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363527078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363527078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:17:47,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363527180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363527189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363527189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363527189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363527189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363527389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363527400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363527400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363527401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363527404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:17:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,424 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e15e1f532fb540259abbc86f2dea693c 2024-11-11T22:17:47,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c90434fdbf6f43f1bfe762b992e26906 is 50, key is test_row_0/C:col10/1731363466429/Put/seqid=0 2024-11-11T22:17:47,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742130_1306 (size=12301) 2024-11-11T22:17:47,486 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c90434fdbf6f43f1bfe762b992e26906 2024-11-11T22:17:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ae80cc87601446bda92bd415f3644b84 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ae80cc87601446bda92bd415f3644b84 2024-11-11T22:17:47,503 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ae80cc87601446bda92bd415f3644b84, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T22:17:47,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/e15e1f532fb540259abbc86f2dea693c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e15e1f532fb540259abbc86f2dea693c 2024-11-11T22:17:47,513 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e15e1f532fb540259abbc86f2dea693c, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T22:17:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c90434fdbf6f43f1bfe762b992e26906 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c90434fdbf6f43f1bfe762b992e26906 2024-11-11T22:17:47,519 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c90434fdbf6f43f1bfe762b992e26906, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T22:17:47,520 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for aea091a0b91074e84f623acef361ebf8 in 586ms, sequenceid=408, compaction requested=true 2024-11-11T22:17:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:47,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:47,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T22:17:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T22:17:47,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T22:17:47,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 740 msec 2024-11-11T22:17:47,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 745 msec 2024-11-11T22:17:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:47,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:17:47,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:47,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:47,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:47,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:47,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:47,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:47,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ed106079161b4752ba07b787bf004895 is 50, key is test_row_0/A:col10/1731363467076/Put/seqid=0 2024-11-11T22:17:47,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742131_1307 (size=14741) 2024-11-11T22:17:47,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363527724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363527733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363527733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363527735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363527735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363527840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363527850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363527850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363527850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363527850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:47,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T22:17:47,920 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:47,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T22:17:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:17:47,923 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:47,924 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:47,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:17:48,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363528049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363528060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363528060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363528060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363528060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T22:17:48,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:48,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ed106079161b4752ba07b787bf004895 2024-11-11T22:17:48,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/20d7efc4bc2344f9941e281f15fd69fa is 50, key is test_row_0/B:col10/1731363467076/Put/seqid=0 2024-11-11T22:17:48,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742132_1308 (size=12301) 2024-11-11T22:17:48,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T22:17:48,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:48,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:17:48,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363528359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363528370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363528370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363528370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363528370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T22:17:48,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:48,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T22:17:48,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:48,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:17:48,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/20d7efc4bc2344f9941e281f15fd69fa 2024-11-11T22:17:48,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/4ba6079b8841402bb30a95a5d08e3b16 is 50, key is test_row_0/C:col10/1731363467076/Put/seqid=0 2024-11-11T22:17:48,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742133_1309 (size=12301) 2024-11-11T22:17:48,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/4ba6079b8841402bb30a95a5d08e3b16 2024-11-11T22:17:48,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/ed106079161b4752ba07b787bf004895 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ed106079161b4752ba07b787bf004895 2024-11-11T22:17:48,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ed106079161b4752ba07b787bf004895, entries=200, sequenceid=427, filesize=14.4 K 2024-11-11T22:17:48,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/20d7efc4bc2344f9941e281f15fd69fa as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/20d7efc4bc2344f9941e281f15fd69fa 2024-11-11T22:17:48,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/20d7efc4bc2344f9941e281f15fd69fa, entries=150, sequenceid=427, filesize=12.0 K 2024-11-11T22:17:48,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/4ba6079b8841402bb30a95a5d08e3b16 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4ba6079b8841402bb30a95a5d08e3b16 2024-11-11T22:17:48,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4ba6079b8841402bb30a95a5d08e3b16, entries=150, sequenceid=427, filesize=12.0 K 2024-11-11T22:17:48,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for aea091a0b91074e84f623acef361ebf8 in 967ms, sequenceid=427, compaction requested=true 2024-11-11T22:17:48,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:48,672 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:48,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:48,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:48,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:48,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:48,672 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:48,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:48,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:48,674 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:48,674 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:48,674 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,674 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/6a7efd98fbc74b2d8e03d6438688a218, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/31de3ceeab344784a8f10b751c010a6c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ae80cc87601446bda92bd415f3644b84, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ed106079161b4752ba07b787bf004895] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=53.6 K 2024-11-11T22:17:48,676 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:48,676 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7efd98fbc74b2d8e03d6438688a218, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731363464102 2024-11-11T22:17:48,676 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:48,676 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,677 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/273066f6c8d247c5bd4dcd88e338102d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e24abe3703d04d81bfd067159f24a6dd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e15e1f532fb540259abbc86f2dea693c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/20d7efc4bc2344f9941e281f15fd69fa] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=48.8 K 2024-11-11T22:17:48,680 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31de3ceeab344784a8f10b751c010a6c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731363465262 2024-11-11T22:17:48,681 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 273066f6c8d247c5bd4dcd88e338102d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731363464102 2024-11-11T22:17:48,681 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae80cc87601446bda92bd415f3644b84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731363466424 2024-11-11T22:17:48,684 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e24abe3703d04d81bfd067159f24a6dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731363465269 2024-11-11T22:17:48,685 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed106079161b4752ba07b787bf004895, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731363467075 2024-11-11T22:17:48,687 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e15e1f532fb540259abbc86f2dea693c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731363466424 2024-11-11T22:17:48,688 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d7efc4bc2344f9941e281f15fd69fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731363467076 2024-11-11T22:17:48,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T22:17:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:48,690 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T22:17:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:48,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/dff329c548c2448f87fdb1c0c0320efa is 50, key is test_row_0/A:col10/1731363467721/Put/seqid=0 2024-11-11T22:17:48,725 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:48,726 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/bb567264db3f42f9b96eff1ce6dde96d is 50, key is test_row_0/B:col10/1731363467076/Put/seqid=0 2024-11-11T22:17:48,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742134_1310 (size=12301) 2024-11-11T22:17:48,729 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:48,730 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/dff329c548c2448f87fdb1c0c0320efa 2024-11-11T22:17:48,730 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/5ee2a53ac40e4152afcbe091c0bd908a is 50, key is test_row_0/A:col10/1731363467076/Put/seqid=0 2024-11-11T22:17:48,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742136_1312 (size=13255) 2024-11-11T22:17:48,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742135_1311 (size=13255) 2024-11-11T22:17:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/0afe7f5578f244dbb92af1d87b394031 is 50, key is test_row_0/B:col10/1731363467721/Put/seqid=0 2024-11-11T22:17:48,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742137_1313 (size=12301) 2024-11-11T22:17:48,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363528895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363528895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363528897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363528901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363528901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363529009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363529010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363529010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363529010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363529010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:17:49,149 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/bb567264db3f42f9b96eff1ce6dde96d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/bb567264db3f42f9b96eff1ce6dde96d 2024-11-11T22:17:49,160 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/0afe7f5578f244dbb92af1d87b394031 2024-11-11T22:17:49,161 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/5ee2a53ac40e4152afcbe091c0bd908a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/5ee2a53ac40e4152afcbe091c0bd908a 2024-11-11T22:17:49,173 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into bb567264db3f42f9b96eff1ce6dde96d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:49,174 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:49,174 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=12, startTime=1731363468672; duration=0sec 2024-11-11T22:17:49,174 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:49,174 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:49,174 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:17:49,176 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:17:49,176 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:49,176 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:49,176 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/b015b933bed44dbfa9da8ec4b86048a7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8784daf0f03d4e0da729d4b02fe208f3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c90434fdbf6f43f1bfe762b992e26906, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4ba6079b8841402bb30a95a5d08e3b16] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=48.8 K 2024-11-11T22:17:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/8d5a628257714a46a7f57b10fb254938 is 50, key is test_row_0/C:col10/1731363467721/Put/seqid=0 2024-11-11T22:17:49,179 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b015b933bed44dbfa9da8ec4b86048a7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731363464102 2024-11-11T22:17:49,180 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8784daf0f03d4e0da729d4b02fe208f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731363465269 2024-11-11T22:17:49,180 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c90434fdbf6f43f1bfe762b992e26906, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731363466424 2024-11-11T22:17:49,180 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ba6079b8841402bb30a95a5d08e3b16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731363467076 2024-11-11T22:17:49,181 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into 5ee2a53ac40e4152afcbe091c0bd908a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:49,181 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:49,181 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=12, startTime=1731363468672; duration=0sec 2024-11-11T22:17:49,181 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:49,181 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:49,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742138_1314 (size=12301) 2024-11-11T22:17:49,189 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:49,189 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/4fa45929addb477ca05746050368e92a is 50, key is test_row_0/C:col10/1731363467076/Put/seqid=0 2024-11-11T22:17:49,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742139_1315 (size=13255) 2024-11-11T22:17:49,197 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/4fa45929addb477ca05746050368e92a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4fa45929addb477ca05746050368e92a 2024-11-11T22:17:49,201 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into 4fa45929addb477ca05746050368e92a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:49,202 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:49,202 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=12, startTime=1731363468672; duration=0sec 2024-11-11T22:17:49,202 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:49,202 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:49,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363529219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363529219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363529220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363529219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363529220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363529530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363529530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363529530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363529530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363529532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:49,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:49,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:49,584 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/8d5a628257714a46a7f57b10fb254938 2024-11-11T22:17:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/dff329c548c2448f87fdb1c0c0320efa as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dff329c548c2448f87fdb1c0c0320efa 2024-11-11T22:17:49,597 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dff329c548c2448f87fdb1c0c0320efa, entries=150, sequenceid=441, filesize=12.0 K 2024-11-11T22:17:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/0afe7f5578f244dbb92af1d87b394031 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0afe7f5578f244dbb92af1d87b394031 2024-11-11T22:17:49,609 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0afe7f5578f244dbb92af1d87b394031, entries=150, sequenceid=441, filesize=12.0 K 2024-11-11T22:17:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/8d5a628257714a46a7f57b10fb254938 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8d5a628257714a46a7f57b10fb254938 2024-11-11T22:17:49,617 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8d5a628257714a46a7f57b10fb254938, entries=150, sequenceid=441, filesize=12.0 K 2024-11-11T22:17:49,620 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for aea091a0b91074e84f623acef361ebf8 in 930ms, sequenceid=441, compaction requested=false 2024-11-11T22:17:49,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:49,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:49,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T22:17:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T22:17:49,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T22:17:49,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6980 sec 2024-11-11T22:17:49,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7110 sec 2024-11-11T22:17:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:50,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:17:50,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:50,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:50,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:50,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:50,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:50,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:50,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/646453881875484888b24c5ff5c9c0be is 50, key is test_row_0/A:col10/1731363468893/Put/seqid=0 2024-11-11T22:17:50,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742140_1316 (size=12301) 2024-11-11T22:17:50,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/646453881875484888b24c5ff5c9c0be 2024-11-11T22:17:50,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363530052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363530053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363530053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/99633d2d127c4baa990e45b04de9c039 is 50, key is test_row_0/B:col10/1731363468893/Put/seqid=0 2024-11-11T22:17:50,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363530058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363530059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742141_1317 (size=12301) 2024-11-11T22:17:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T22:17:50,069 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:50,070 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T22:17:50,071 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:17:50,072 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:50,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:50,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363530170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363530170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363530170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363530170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363530170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:17:50,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:17:50,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:50,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:17:50,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:50,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363530380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363530380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363530380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363530380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363530380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:17:50,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/99633d2d127c4baa990e45b04de9c039 2024-11-11T22:17:50,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/884d231b789646f58783557ebc470868 is 50, key is test_row_0/C:col10/1731363468893/Put/seqid=0 2024-11-11T22:17:50,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742142_1318 (size=12301) 2024-11-11T22:17:50,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:17:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:17:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363530689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363530690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363530690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363530690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363530690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:50,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:17:50,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:17:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:50,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/884d231b789646f58783557ebc470868 2024-11-11T22:17:50,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/646453881875484888b24c5ff5c9c0be as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/646453881875484888b24c5ff5c9c0be 2024-11-11T22:17:50,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/646453881875484888b24c5ff5c9c0be, entries=150, sequenceid=464, filesize=12.0 K 2024-11-11T22:17:50,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/99633d2d127c4baa990e45b04de9c039 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/99633d2d127c4baa990e45b04de9c039 2024-11-11T22:17:50,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/99633d2d127c4baa990e45b04de9c039, entries=150, sequenceid=464, filesize=12.0 K 2024-11-11T22:17:50,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/884d231b789646f58783557ebc470868 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/884d231b789646f58783557ebc470868 2024-11-11T22:17:50,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/884d231b789646f58783557ebc470868, entries=150, sequenceid=464, filesize=12.0 K 2024-11-11T22:17:50,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for aea091a0b91074e84f623acef361ebf8 in 879ms, sequenceid=464, compaction requested=true 2024-11-11T22:17:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:50,922 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:50,922 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:50,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:50,923 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:50,923 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:50,923 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,923 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/5ee2a53ac40e4152afcbe091c0bd908a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dff329c548c2448f87fdb1c0c0320efa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/646453881875484888b24c5ff5c9c0be] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=37.0 K 2024-11-11T22:17:50,924 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:50,924 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:50,924 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,924 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/bb567264db3f42f9b96eff1ce6dde96d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0afe7f5578f244dbb92af1d87b394031, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/99633d2d127c4baa990e45b04de9c039] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=37.0 K 2024-11-11T22:17:50,924 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bb567264db3f42f9b96eff1ce6dde96d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731363467076 2024-11-11T22:17:50,924 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee2a53ac40e4152afcbe091c0bd908a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731363467076 2024-11-11T22:17:50,925 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0afe7f5578f244dbb92af1d87b394031, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731363467721 2024-11-11T22:17:50,925 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting dff329c548c2448f87fdb1c0c0320efa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731363467721 2024-11-11T22:17:50,925 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 99633d2d127c4baa990e45b04de9c039, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731363468893 2024-11-11T22:17:50,925 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 646453881875484888b24c5ff5c9c0be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731363468893 2024-11-11T22:17:50,942 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:50,943 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/87382515bc35467ba3ff5936d9da7b55 is 50, key is test_row_0/B:col10/1731363468893/Put/seqid=0 2024-11-11T22:17:50,946 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:50,946 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/aeb01b286c664fc7a90451fcba19d755 is 50, key is test_row_0/A:col10/1731363468893/Put/seqid=0 2024-11-11T22:17:50,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742143_1319 (size=13357) 2024-11-11T22:17:50,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T22:17:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:50,997 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:17:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:50,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:51,004 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/87382515bc35467ba3ff5936d9da7b55 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/87382515bc35467ba3ff5936d9da7b55 2024-11-11T22:17:51,014 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into 87382515bc35467ba3ff5936d9da7b55(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:51,014 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:51,014 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=13, startTime=1731363470922; duration=0sec 2024-11-11T22:17:51,014 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:51,014 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:51,014 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:51,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/dd0dc271501245a3b18702d2dd15bd62 is 50, key is test_row_0/A:col10/1731363470048/Put/seqid=0 2024-11-11T22:17:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742144_1320 (size=13357) 2024-11-11T22:17:51,019 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:51,019 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:51,020 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:51,020 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4fa45929addb477ca05746050368e92a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8d5a628257714a46a7f57b10fb254938, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/884d231b789646f58783557ebc470868] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=37.0 K 2024-11-11T22:17:51,023 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fa45929addb477ca05746050368e92a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731363467076 2024-11-11T22:17:51,024 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5a628257714a46a7f57b10fb254938, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731363467721 2024-11-11T22:17:51,025 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/aeb01b286c664fc7a90451fcba19d755 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/aeb01b286c664fc7a90451fcba19d755 2024-11-11T22:17:51,025 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 884d231b789646f58783557ebc470868, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731363468893 2024-11-11T22:17:51,032 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into aeb01b286c664fc7a90451fcba19d755(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:51,032 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:51,032 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=13, startTime=1731363470922; duration=0sec 2024-11-11T22:17:51,032 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:51,032 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:51,067 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#279 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:51,067 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/91dc23f0ffc744fcb112f658d16c45a5 is 50, key is test_row_0/C:col10/1731363468893/Put/seqid=0 2024-11-11T22:17:51,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742145_1321 (size=12301) 2024-11-11T22:17:51,090 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/dd0dc271501245a3b18702d2dd15bd62 2024-11-11T22:17:51,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742146_1322 (size=13357) 2024-11-11T22:17:51,143 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/91dc23f0ffc744fcb112f658d16c45a5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/91dc23f0ffc744fcb112f658d16c45a5 2024-11-11T22:17:51,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/3817fd5255054269b7ad81baa36f8f1f is 50, key is test_row_0/B:col10/1731363470048/Put/seqid=0 2024-11-11T22:17:51,149 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into 91dc23f0ffc744fcb112f658d16c45a5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:51,149 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:51,149 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=13, startTime=1731363470922; duration=0sec 2024-11-11T22:17:51,149 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:51,149 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:51,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742147_1323 (size=12301) 2024-11-11T22:17:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:51,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:17:51,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363531230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363531230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363531231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363531231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363531231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363531340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363531340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363531340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363531344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363531344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363531549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363531549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363531549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,559 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/3817fd5255054269b7ad81baa36f8f1f 2024-11-11T22:17:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363531560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363531560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/fec0bcfcc02c4dcdafb0fceaef76f096 is 50, key is test_row_0/C:col10/1731363470048/Put/seqid=0 2024-11-11T22:17:51,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742148_1324 (size=12301) 2024-11-11T22:17:51,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363531860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363531860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363531860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363531869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363531869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:51,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:51,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:51,970 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/fec0bcfcc02c4dcdafb0fceaef76f096 2024-11-11T22:17:51,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/dd0dc271501245a3b18702d2dd15bd62 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dd0dc271501245a3b18702d2dd15bd62 2024-11-11T22:17:51,984 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dd0dc271501245a3b18702d2dd15bd62, entries=150, sequenceid=477, filesize=12.0 K 2024-11-11T22:17:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/3817fd5255054269b7ad81baa36f8f1f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3817fd5255054269b7ad81baa36f8f1f 2024-11-11T22:17:51,991 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3817fd5255054269b7ad81baa36f8f1f, entries=150, sequenceid=477, filesize=12.0 K 2024-11-11T22:17:51,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/fec0bcfcc02c4dcdafb0fceaef76f096 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/fec0bcfcc02c4dcdafb0fceaef76f096 2024-11-11T22:17:52,000 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/fec0bcfcc02c4dcdafb0fceaef76f096, entries=150, sequenceid=477, filesize=12.0 K 2024-11-11T22:17:52,009 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aea091a0b91074e84f623acef361ebf8 in 1011ms, sequenceid=477, compaction requested=false 2024-11-11T22:17:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T22:17:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T22:17:52,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T22:17:52,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9420 sec 2024-11-11T22:17:52,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.9490 sec 2024-11-11T22:17:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T22:17:52,219 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:52,220 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:17:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-11T22:17:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:17:52,222 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:17:52,223 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:17:52,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:17:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:17:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:52,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:17:52,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:52,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:52,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:52,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:52,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:52,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:17:52,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/477e563d7d6747a8981826fe7e666b57 is 50, key is test_row_0/A:col10/1731363472376/Put/seqid=0 2024-11-11T22:17:52,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363532402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363532404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363532405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363532406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363532409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742149_1325 (size=12301) 2024-11-11T22:17:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363532520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363532520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363532520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363532520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363532520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:52,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:52,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:17:52,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363532730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363532730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363532730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363532730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363532730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:52,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/477e563d7d6747a8981826fe7e666b57 2024-11-11T22:17:52,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/65a5c53ff1274bf09afa3c4b8e903264 is 50, key is test_row_0/B:col10/1731363472376/Put/seqid=0 2024-11-11T22:17:52,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742150_1326 (size=12301) 2024-11-11T22:17:52,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:52,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:52,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:17:52,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:52,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:52,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:52,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363533040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363533040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363533040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363533040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363533040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,107 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-11T22:17:53,107 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:53,107 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,107 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:53,107 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-11T22:17:53,107 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:53,107 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,107 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:53,110 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-11T22:17:53,110 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-11-11T22:17:53,110 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:53,110 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:53,110 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,110 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,111 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,111 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:53,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:53,112 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-11T22:17:53,112 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:53,112 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,112 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:53,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:53,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:53,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:53,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/65a5c53ff1274bf09afa3c4b8e903264 2024-11-11T22:17:53,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c56c21d2e82040e3a9c0b1f31b7cad2f is 50, key is test_row_0/C:col10/1731363472376/Put/seqid=0 2024-11-11T22:17:53,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742151_1327 (size=12301) 2024-11-11T22:17:53,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:17:53,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:53,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:53,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53642 deadline: 1731363533550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53658 deadline: 1731363533550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53704 deadline: 1731363533550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53622 deadline: 1731363533550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:17:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53664 deadline: 1731363533559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:17:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:17:53,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:53,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:53,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:17:53,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c56c21d2e82040e3a9c0b1f31b7cad2f 2024-11-11T22:17:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/477e563d7d6747a8981826fe7e666b57 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/477e563d7d6747a8981826fe7e666b57 2024-11-11T22:17:53,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/477e563d7d6747a8981826fe7e666b57, entries=150, sequenceid=501, filesize=12.0 K 2024-11-11T22:17:53,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/65a5c53ff1274bf09afa3c4b8e903264 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/65a5c53ff1274bf09afa3c4b8e903264 2024-11-11T22:17:53,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/65a5c53ff1274bf09afa3c4b8e903264, entries=150, sequenceid=501, filesize=12.0 K 2024-11-11T22:17:53,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/c56c21d2e82040e3a9c0b1f31b7cad2f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c56c21d2e82040e3a9c0b1f31b7cad2f 2024-11-11T22:17:53,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c56c21d2e82040e3a9c0b1f31b7cad2f, entries=150, sequenceid=501, filesize=12.0 K 2024-11-11T22:17:53,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for aea091a0b91074e84f623acef361ebf8 in 1293ms, sequenceid=501, compaction requested=true 2024-11-11T22:17:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:17:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:17:53,670 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aea091a0b91074e84f623acef361ebf8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:17:53,670 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:53,671 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:53,671 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:53,671 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/B is initiating minor compaction (all files) 2024-11-11T22:17:53,671 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/A is initiating minor compaction (all files) 2024-11-11T22:17:53,671 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/B in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,671 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/A in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,671 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/87382515bc35467ba3ff5936d9da7b55, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3817fd5255054269b7ad81baa36f8f1f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/65a5c53ff1274bf09afa3c4b8e903264] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=37.1 K 2024-11-11T22:17:53,671 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/aeb01b286c664fc7a90451fcba19d755, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dd0dc271501245a3b18702d2dd15bd62, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/477e563d7d6747a8981826fe7e666b57] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=37.1 K 2024-11-11T22:17:53,671 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 87382515bc35467ba3ff5936d9da7b55, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731363468893 2024-11-11T22:17:53,671 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb01b286c664fc7a90451fcba19d755, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731363468893 2024-11-11T22:17:53,671 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3817fd5255054269b7ad81baa36f8f1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731363470048 2024-11-11T22:17:53,671 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd0dc271501245a3b18702d2dd15bd62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731363470048 2024-11-11T22:17:53,672 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 477e563d7d6747a8981826fe7e666b57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731363471229 2024-11-11T22:17:53,672 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 65a5c53ff1274bf09afa3c4b8e903264, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731363471229 2024-11-11T22:17:53,680 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#A#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:53,680 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#B#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:53,680 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/48ed8e5b2d4146adad696e9b67d8a085 is 50, key is test_row_0/B:col10/1731363472376/Put/seqid=0 2024-11-11T22:17:53,680 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/12a756f0308b4632b7d22962c7402288 is 50, key is test_row_0/A:col10/1731363472376/Put/seqid=0 2024-11-11T22:17:53,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742153_1329 (size=13459) 2024-11-11T22:17:53,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742152_1328 (size=13459) 2024-11-11T22:17:53,689 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/12a756f0308b4632b7d22962c7402288 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/12a756f0308b4632b7d22962c7402288 2024-11-11T22:17:53,694 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/A of aea091a0b91074e84f623acef361ebf8 into 12a756f0308b4632b7d22962c7402288(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:53,694 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:53,694 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/A, priority=13, startTime=1731363473669; duration=0sec 2024-11-11T22:17:53,694 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:17:53,694 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:A 2024-11-11T22:17:53,694 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:17:53,695 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:17:53,695 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): aea091a0b91074e84f623acef361ebf8/C is initiating minor compaction (all files) 2024-11-11T22:17:53,695 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aea091a0b91074e84f623acef361ebf8/C in TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,695 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/91dc23f0ffc744fcb112f658d16c45a5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/fec0bcfcc02c4dcdafb0fceaef76f096, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c56c21d2e82040e3a9c0b1f31b7cad2f] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp, totalSize=37.1 K 2024-11-11T22:17:53,695 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91dc23f0ffc744fcb112f658d16c45a5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1731363468893 2024-11-11T22:17:53,696 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fec0bcfcc02c4dcdafb0fceaef76f096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731363470048 2024-11-11T22:17:53,696 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c56c21d2e82040e3a9c0b1f31b7cad2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731363471229 2024-11-11T22:17:53,701 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aea091a0b91074e84f623acef361ebf8#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:17:53,702 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/8da746cc4ad84b1b9e23d2694b60cf72 is 50, key is test_row_0/C:col10/1731363472376/Put/seqid=0 2024-11-11T22:17:53,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742154_1330 (size=13459) 2024-11-11T22:17:53,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T22:17:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:53,763 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:17:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:53,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/11829271f9d54be8b60762af8a797976 is 50, key is test_row_0/A:col10/1731363472405/Put/seqid=0 2024-11-11T22:17:53,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742155_1331 (size=12301) 2024-11-11T22:17:54,088 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/48ed8e5b2d4146adad696e9b67d8a085 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/48ed8e5b2d4146adad696e9b67d8a085 2024-11-11T22:17:54,093 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/B of aea091a0b91074e84f623acef361ebf8 into 48ed8e5b2d4146adad696e9b67d8a085(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:54,093 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:54,093 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/B, priority=13, startTime=1731363473669; duration=0sec 2024-11-11T22:17:54,093 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:54,093 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:B 2024-11-11T22:17:54,109 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/8da746cc4ad84b1b9e23d2694b60cf72 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8da746cc4ad84b1b9e23d2694b60cf72 2024-11-11T22:17:54,114 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aea091a0b91074e84f623acef361ebf8/C of aea091a0b91074e84f623acef361ebf8 into 8da746cc4ad84b1b9e23d2694b60cf72(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:17:54,114 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:54,114 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., storeName=aea091a0b91074e84f623acef361ebf8/C, priority=13, startTime=1731363473670; duration=0sec 2024-11-11T22:17:54,114 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:17:54,114 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aea091a0b91074e84f623acef361ebf8:C 2024-11-11T22:17:54,171 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/11829271f9d54be8b60762af8a797976 2024-11-11T22:17:54,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/19e45a9025674d8a9d51d8a049568e66 is 50, key is test_row_0/B:col10/1731363472405/Put/seqid=0 2024-11-11T22:17:54,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742156_1332 (size=12301) 2024-11-11T22:17:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:17:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:54,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. as already flushing 2024-11-11T22:17:54,562 INFO [Thread-1064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1064. 2024-11-11T22:17:54,562 DEBUG [Thread-1064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:54,562 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,562 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-11T22:17:54,562 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:54,562 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,562 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:54,562 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,562 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-11T22:17:54,562 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:54,562 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:54,562 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:54,563 INFO [Thread-1066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1066. 2024-11-11T22:17:54,563 DEBUG [Thread-1066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:54,563 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,563 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:54,581 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-11-11T22:17:54,581 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:17:54,581 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,582 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:54,582 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:54,582 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/19e45a9025674d8a9d51d8a049568e66 2024-11-11T22:17:54,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/cd212ca0d7934dd8b8d73954338ceca9 is 50, key is test_row_0/C:col10/1731363472405/Put/seqid=0 2024-11-11T22:17:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742157_1333 (size=12301) 2024-11-11T22:17:54,605 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/cd212ca0d7934dd8b8d73954338ceca9 2024-11-11T22:17:54,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/11829271f9d54be8b60762af8a797976 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11829271f9d54be8b60762af8a797976 2024-11-11T22:17:54,617 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11829271f9d54be8b60762af8a797976, entries=150, sequenceid=514, filesize=12.0 K 2024-11-11T22:17:54,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/19e45a9025674d8a9d51d8a049568e66 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/19e45a9025674d8a9d51d8a049568e66 2024-11-11T22:17:54,623 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/19e45a9025674d8a9d51d8a049568e66, entries=150, sequenceid=514, filesize=12.0 K 2024-11-11T22:17:54,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/cd212ca0d7934dd8b8d73954338ceca9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/cd212ca0d7934dd8b8d73954338ceca9 2024-11-11T22:17:54,628 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/cd212ca0d7934dd8b8d73954338ceca9, entries=150, sequenceid=514, filesize=12.0 K 2024-11-11T22:17:54,628 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=33.54 KB/34350 for aea091a0b91074e84f623acef361ebf8 in 865ms, sequenceid=514, compaction requested=false 2024-11-11T22:17:54,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for aea091a0b91074e84f623acef361ebf8: 2024-11-11T22:17:54,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:54,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-11T22:17:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-11T22:17:54,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T22:17:54,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4070 sec 2024-11-11T22:17:54,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.4120 sec 2024-11-11T22:17:56,250 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:17:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T22:17:56,391 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:56,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:17:56,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-11T22:17:56,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-11T22:17:56,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 221 2024-11-11T22:17:56,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-11T22:17:56,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T22:17:56,391 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:17:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2486 2024-11-11T22:17:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2373 2024-11-11T22:17:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2347 2024-11-11T22:17:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2505 2024-11-11T22:17:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2403 2024-11-11T22:17:56,392 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:17:56,392 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:17:56,392 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:17:56,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:56,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:17:56,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:17:56,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:17:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T22:17:56,400 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363476399"}]},"ts":"1731363476399"} 2024-11-11T22:17:56,401 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:17:56,401 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:17:56,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:17:56,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, UNASSIGN}] 2024-11-11T22:17:56,403 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, UNASSIGN 2024-11-11T22:17:56,404 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=aea091a0b91074e84f623acef361ebf8, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:56,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, UNASSIGN because future has completed 2024-11-11T22:17:56,405 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:17:56,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:17:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T22:17:56,558 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:56,558 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:17:56,558 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing aea091a0b91074e84f623acef361ebf8, disabling compactions & flushes 2024-11-11T22:17:56,558 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:56,558 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:56,558 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. after waiting 0 ms 2024-11-11T22:17:56,558 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:56,559 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing aea091a0b91074e84f623acef361ebf8 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T22:17:56,559 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=A 2024-11-11T22:17:56,559 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:56,559 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=B 2024-11-11T22:17:56,559 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:56,559 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aea091a0b91074e84f623acef361ebf8, store=C 2024-11-11T22:17:56,559 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:17:56,563 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/8fa623e79f724a74911ce7386974bbb6 is 50, key is test_row_0/A:col10/1731363474561/Put/seqid=0 2024-11-11T22:17:56,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742158_1334 (size=12301) 2024-11-11T22:17:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T22:17:56,966 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/8fa623e79f724a74911ce7386974bbb6 2024-11-11T22:17:56,974 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/b1b939cb63144a0ea8bb89ffa5406ef5 is 50, key is test_row_0/B:col10/1731363474561/Put/seqid=0 2024-11-11T22:17:56,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742159_1335 (size=12301) 2024-11-11T22:17:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T22:17:57,379 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/b1b939cb63144a0ea8bb89ffa5406ef5 2024-11-11T22:17:57,385 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/3b8a9d1d8cd9409280956c3636ee2f11 is 50, key is test_row_0/C:col10/1731363474561/Put/seqid=0 2024-11-11T22:17:57,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742160_1336 (size=12301) 2024-11-11T22:17:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T22:17:57,788 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/3b8a9d1d8cd9409280956c3636ee2f11 2024-11-11T22:17:57,792 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/A/8fa623e79f724a74911ce7386974bbb6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/8fa623e79f724a74911ce7386974bbb6 2024-11-11T22:17:57,796 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/8fa623e79f724a74911ce7386974bbb6, entries=150, sequenceid=524, filesize=12.0 K 2024-11-11T22:17:57,796 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/B/b1b939cb63144a0ea8bb89ffa5406ef5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/b1b939cb63144a0ea8bb89ffa5406ef5 2024-11-11T22:17:57,801 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/b1b939cb63144a0ea8bb89ffa5406ef5, entries=150, sequenceid=524, filesize=12.0 K 2024-11-11T22:17:57,802 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/.tmp/C/3b8a9d1d8cd9409280956c3636ee2f11 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3b8a9d1d8cd9409280956c3636ee2f11 2024-11-11T22:17:57,806 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3b8a9d1d8cd9409280956c3636ee2f11, entries=150, sequenceid=524, filesize=12.0 K 2024-11-11T22:17:57,806 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for aea091a0b91074e84f623acef361ebf8 in 1248ms, sequenceid=524, compaction requested=true 2024-11-11T22:17:57,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/96e52b36a36f4d07af5f5478e57ddc9c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/acbec3bdb3a147a7942425b8a03b8c60, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c434bc9f4c2f416cb1dee8fb3d5f0a02, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/63734e21f5da410aa14f5aeefa375536, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c421ae42e9514218ad3f8ceab77a31cb, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1cf44b5dafbf489db2ef1be8e7955db3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b1d5e0bf3f234375ada307085ac9e0a2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cc5ccdcee13946eeb6db29586d79f823, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1716672cce5c41f3a1ae624778a28180, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ccfa3c76fb4a48b383d2c658079d63c8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/54998f0f3e1b4aa89a56c9fd438cae89, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ca674c7f7a5a43faaabfd1496ccb7cf8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cb7cf42143ce4ea6b3761a05e19cbeb3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ab477fb548534705a337f922413f695f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/fe22e98f4c4d481cb3fb8ff9ab5cf817, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/2a3286d106ad447eabb83501048bfe66, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/48522a08b46941409ddc2adfe7fbbd2b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11dd7054c508410eb3a4f9758bb23a3e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/620144088bb6417886a72254b230a4ae, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b11087115b31437cb88e0fd2a7b457ac, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/47780e288dbd4a6191342bf753da0a96, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/67958ac9d0ff49b7a8170de3bc08d9b3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/4ccaccdfda8b4866b4c43162bcea464e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/066aab1904f34aa9959bcd5a45ba018e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/6a7efd98fbc74b2d8e03d6438688a218, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/31de3ceeab344784a8f10b751c010a6c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ae80cc87601446bda92bd415f3644b84, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ed106079161b4752ba07b787bf004895, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/5ee2a53ac40e4152afcbe091c0bd908a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dff329c548c2448f87fdb1c0c0320efa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/aeb01b286c664fc7a90451fcba19d755, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/646453881875484888b24c5ff5c9c0be, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dd0dc271501245a3b18702d2dd15bd62, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/477e563d7d6747a8981826fe7e666b57] to archive 2024-11-11T22:17:57,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:57,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/96e52b36a36f4d07af5f5478e57ddc9c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/96e52b36a36f4d07af5f5478e57ddc9c 2024-11-11T22:17:57,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/acbec3bdb3a147a7942425b8a03b8c60 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/acbec3bdb3a147a7942425b8a03b8c60 2024-11-11T22:17:57,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c434bc9f4c2f416cb1dee8fb3d5f0a02 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c434bc9f4c2f416cb1dee8fb3d5f0a02 2024-11-11T22:17:57,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/63734e21f5da410aa14f5aeefa375536 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/63734e21f5da410aa14f5aeefa375536 2024-11-11T22:17:57,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c421ae42e9514218ad3f8ceab77a31cb to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/c421ae42e9514218ad3f8ceab77a31cb 2024-11-11T22:17:57,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1cf44b5dafbf489db2ef1be8e7955db3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1cf44b5dafbf489db2ef1be8e7955db3 2024-11-11T22:17:57,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b1d5e0bf3f234375ada307085ac9e0a2 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b1d5e0bf3f234375ada307085ac9e0a2 2024-11-11T22:17:57,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cc5ccdcee13946eeb6db29586d79f823 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cc5ccdcee13946eeb6db29586d79f823 2024-11-11T22:17:57,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1716672cce5c41f3a1ae624778a28180 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/1716672cce5c41f3a1ae624778a28180 2024-11-11T22:17:57,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ccfa3c76fb4a48b383d2c658079d63c8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ccfa3c76fb4a48b383d2c658079d63c8 2024-11-11T22:17:57,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/54998f0f3e1b4aa89a56c9fd438cae89 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/54998f0f3e1b4aa89a56c9fd438cae89 2024-11-11T22:17:57,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ca674c7f7a5a43faaabfd1496ccb7cf8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ca674c7f7a5a43faaabfd1496ccb7cf8 2024-11-11T22:17:57,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cb7cf42143ce4ea6b3761a05e19cbeb3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/cb7cf42143ce4ea6b3761a05e19cbeb3 2024-11-11T22:17:57,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ab477fb548534705a337f922413f695f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ab477fb548534705a337f922413f695f 2024-11-11T22:17:57,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/fe22e98f4c4d481cb3fb8ff9ab5cf817 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/fe22e98f4c4d481cb3fb8ff9ab5cf817 2024-11-11T22:17:57,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/2a3286d106ad447eabb83501048bfe66 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/2a3286d106ad447eabb83501048bfe66 2024-11-11T22:17:57,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/48522a08b46941409ddc2adfe7fbbd2b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/48522a08b46941409ddc2adfe7fbbd2b 2024-11-11T22:17:57,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11dd7054c508410eb3a4f9758bb23a3e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11dd7054c508410eb3a4f9758bb23a3e 2024-11-11T22:17:57,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/620144088bb6417886a72254b230a4ae to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/620144088bb6417886a72254b230a4ae 2024-11-11T22:17:57,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b11087115b31437cb88e0fd2a7b457ac to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/b11087115b31437cb88e0fd2a7b457ac 2024-11-11T22:17:57,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/47780e288dbd4a6191342bf753da0a96 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/47780e288dbd4a6191342bf753da0a96 2024-11-11T22:17:57,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/67958ac9d0ff49b7a8170de3bc08d9b3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/67958ac9d0ff49b7a8170de3bc08d9b3 2024-11-11T22:17:57,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/4ccaccdfda8b4866b4c43162bcea464e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/4ccaccdfda8b4866b4c43162bcea464e 2024-11-11T22:17:57,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/066aab1904f34aa9959bcd5a45ba018e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/066aab1904f34aa9959bcd5a45ba018e 2024-11-11T22:17:57,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/6a7efd98fbc74b2d8e03d6438688a218 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/6a7efd98fbc74b2d8e03d6438688a218 2024-11-11T22:17:57,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/31de3ceeab344784a8f10b751c010a6c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/31de3ceeab344784a8f10b751c010a6c 2024-11-11T22:17:57,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ae80cc87601446bda92bd415f3644b84 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ae80cc87601446bda92bd415f3644b84 2024-11-11T22:17:57,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ed106079161b4752ba07b787bf004895 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/ed106079161b4752ba07b787bf004895 2024-11-11T22:17:57,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/5ee2a53ac40e4152afcbe091c0bd908a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/5ee2a53ac40e4152afcbe091c0bd908a 2024-11-11T22:17:57,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dff329c548c2448f87fdb1c0c0320efa to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dff329c548c2448f87fdb1c0c0320efa 2024-11-11T22:17:57,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/aeb01b286c664fc7a90451fcba19d755 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/aeb01b286c664fc7a90451fcba19d755 2024-11-11T22:17:57,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/646453881875484888b24c5ff5c9c0be to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/646453881875484888b24c5ff5c9c0be 2024-11-11T22:17:57,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dd0dc271501245a3b18702d2dd15bd62 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/dd0dc271501245a3b18702d2dd15bd62 2024-11-11T22:17:57,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/477e563d7d6747a8981826fe7e666b57 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/477e563d7d6747a8981826fe7e666b57 2024-11-11T22:17:57,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/35cf2ef957014d67ac77b34d0c798702, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/5e30f69d6368485099a159be827babb4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0677e50024a74d67bb4794bbd9640f5f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7bb6f162a45b4a3e9fc180a4aca210df, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/9dc6fbe5ca9e4b058663e18c9c75f016, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/23e16f1d817544328f11703f3d2ea162, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/51c8033a773046308fe4173213f33639, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/559160a6df0f49edaac2c1792c775a16, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e61e2c94c3a242b7aa3a0736f25f24aa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/2d19b42c4ca34b33a5cef21b2a85542c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/840bb8a7f8914a9f96d8c9a9481bff63, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/404978326745416d93d4d44ff85b4d22, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7e5c65055ec94666bc769be20d68c643, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7a13d1af7e2e42399f6c934cb9a1c71b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/34fd705fb139489d8574d86beb89f399, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/40cbcce9b61f4d9f9a5e8047f8a0607c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/c59fedabc26249b49140a9d32619d23d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/f91b6a06bc624020b98fffee8984d629, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3ebeb045cf134e52950671919b46f8e7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/ad2474b5a8f343f880803d0998d9f149, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0bf394a86c084ea3981eb640587520ba, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e954567db4a54e1ba51b77d9b14b08ba, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/273066f6c8d247c5bd4dcd88e338102d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/4c47da336646434dbfca872b4248ce55, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e24abe3703d04d81bfd067159f24a6dd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e15e1f532fb540259abbc86f2dea693c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/bb567264db3f42f9b96eff1ce6dde96d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/20d7efc4bc2344f9941e281f15fd69fa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0afe7f5578f244dbb92af1d87b394031, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/87382515bc35467ba3ff5936d9da7b55, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/99633d2d127c4baa990e45b04de9c039, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3817fd5255054269b7ad81baa36f8f1f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/65a5c53ff1274bf09afa3c4b8e903264] to archive 2024-11-11T22:17:57,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:57,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/35cf2ef957014d67ac77b34d0c798702 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/35cf2ef957014d67ac77b34d0c798702 2024-11-11T22:17:57,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/77c7e3c9bc8f47e39dfb5fb3c5bcb4cf 2024-11-11T22:17:57,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/5e30f69d6368485099a159be827babb4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/5e30f69d6368485099a159be827babb4 2024-11-11T22:17:57,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0677e50024a74d67bb4794bbd9640f5f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0677e50024a74d67bb4794bbd9640f5f 2024-11-11T22:17:57,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7bb6f162a45b4a3e9fc180a4aca210df to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7bb6f162a45b4a3e9fc180a4aca210df 2024-11-11T22:17:57,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/9dc6fbe5ca9e4b058663e18c9c75f016 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/9dc6fbe5ca9e4b058663e18c9c75f016 2024-11-11T22:17:57,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/23e16f1d817544328f11703f3d2ea162 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/23e16f1d817544328f11703f3d2ea162 2024-11-11T22:17:57,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/51c8033a773046308fe4173213f33639 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/51c8033a773046308fe4173213f33639 2024-11-11T22:17:57,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/559160a6df0f49edaac2c1792c775a16 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/559160a6df0f49edaac2c1792c775a16 2024-11-11T22:17:57,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e61e2c94c3a242b7aa3a0736f25f24aa to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e61e2c94c3a242b7aa3a0736f25f24aa 2024-11-11T22:17:57,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/2d19b42c4ca34b33a5cef21b2a85542c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/2d19b42c4ca34b33a5cef21b2a85542c 2024-11-11T22:17:57,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/840bb8a7f8914a9f96d8c9a9481bff63 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/840bb8a7f8914a9f96d8c9a9481bff63 2024-11-11T22:17:57,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/404978326745416d93d4d44ff85b4d22 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/404978326745416d93d4d44ff85b4d22 2024-11-11T22:17:57,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7e5c65055ec94666bc769be20d68c643 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7e5c65055ec94666bc769be20d68c643 2024-11-11T22:17:57,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7a13d1af7e2e42399f6c934cb9a1c71b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/7a13d1af7e2e42399f6c934cb9a1c71b 2024-11-11T22:17:57,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/34fd705fb139489d8574d86beb89f399 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/34fd705fb139489d8574d86beb89f399 2024-11-11T22:17:57,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/40cbcce9b61f4d9f9a5e8047f8a0607c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/40cbcce9b61f4d9f9a5e8047f8a0607c 2024-11-11T22:17:57,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/c59fedabc26249b49140a9d32619d23d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/c59fedabc26249b49140a9d32619d23d 2024-11-11T22:17:57,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/f91b6a06bc624020b98fffee8984d629 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/f91b6a06bc624020b98fffee8984d629 2024-11-11T22:17:57,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3ebeb045cf134e52950671919b46f8e7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3ebeb045cf134e52950671919b46f8e7 2024-11-11T22:17:57,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/ad2474b5a8f343f880803d0998d9f149 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/ad2474b5a8f343f880803d0998d9f149 2024-11-11T22:17:57,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0bf394a86c084ea3981eb640587520ba to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0bf394a86c084ea3981eb640587520ba 2024-11-11T22:17:57,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e954567db4a54e1ba51b77d9b14b08ba to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e954567db4a54e1ba51b77d9b14b08ba 2024-11-11T22:17:57,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/273066f6c8d247c5bd4dcd88e338102d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/273066f6c8d247c5bd4dcd88e338102d 2024-11-11T22:17:57,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/4c47da336646434dbfca872b4248ce55 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/4c47da336646434dbfca872b4248ce55 2024-11-11T22:17:57,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e24abe3703d04d81bfd067159f24a6dd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e24abe3703d04d81bfd067159f24a6dd 2024-11-11T22:17:57,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e15e1f532fb540259abbc86f2dea693c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/e15e1f532fb540259abbc86f2dea693c 2024-11-11T22:17:57,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/bb567264db3f42f9b96eff1ce6dde96d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/bb567264db3f42f9b96eff1ce6dde96d 2024-11-11T22:17:57,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/20d7efc4bc2344f9941e281f15fd69fa to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/20d7efc4bc2344f9941e281f15fd69fa 2024-11-11T22:17:57,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0afe7f5578f244dbb92af1d87b394031 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/0afe7f5578f244dbb92af1d87b394031 2024-11-11T22:17:57,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/87382515bc35467ba3ff5936d9da7b55 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/87382515bc35467ba3ff5936d9da7b55 2024-11-11T22:17:57,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/99633d2d127c4baa990e45b04de9c039 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/99633d2d127c4baa990e45b04de9c039 2024-11-11T22:17:57,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3817fd5255054269b7ad81baa36f8f1f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/3817fd5255054269b7ad81baa36f8f1f 2024-11-11T22:17:57,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/65a5c53ff1274bf09afa3c4b8e903264 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/65a5c53ff1274bf09afa3c4b8e903264 2024-11-11T22:17:57,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/9452793a28e34cc4a95289f4f67c1581, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/7890f1c50dee4ad1b43b91e5929b34cf, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4f92a30b84574f729cf05d8eb9149a78, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/63000643abe44e90b49d0f8dab0b34b1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d71bc205ea4f49f085e6d1a8de761fc1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ee9909a346ae46aaa0c34a85ca679744, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3474dc378d7e46b6b8102377b6d4c6b9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/20c2856b199e44e2b9ee4ca546b92bf8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5bdcaf87d4ef4b85b14b5faeb4916a84, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/eb5d96a991ba4d659099750344ab1e00, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ed26976aec9144ca84dd47d035b92031, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5dee4717845547ad929837d5572a7070, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d20658a2e79b49d99d5acac20ea91e48, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/80d8cbf875db422089cf320a146ace58, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f89e31bad0134fbab7230d9fa3da8da3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/996ba58ec9b74a8ea37d7dce6ae6eb0b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/65e7e3d2340848b88d242652fdf25197, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/909c34022619415897f06842e0bc37b9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/286d7b247c264be58661d9d034966c1c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/993e9b4b40254d738fe4dfc0687cf6a4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c685facbcb0647bd867e9b62a7d0329b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ab385c7b1c234d5eb94ac39b53262a45, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/b015b933bed44dbfa9da8ec4b86048a7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3471a1caf93145d1bea000c8ca5a9950, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8784daf0f03d4e0da729d4b02fe208f3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c90434fdbf6f43f1bfe762b992e26906, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4fa45929addb477ca05746050368e92a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4ba6079b8841402bb30a95a5d08e3b16, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8d5a628257714a46a7f57b10fb254938, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/91dc23f0ffc744fcb112f658d16c45a5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/884d231b789646f58783557ebc470868, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/fec0bcfcc02c4dcdafb0fceaef76f096, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c56c21d2e82040e3a9c0b1f31b7cad2f] to archive 2024-11-11T22:17:57,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:17:57,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/9452793a28e34cc4a95289f4f67c1581 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/9452793a28e34cc4a95289f4f67c1581 2024-11-11T22:17:57,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/7890f1c50dee4ad1b43b91e5929b34cf to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/7890f1c50dee4ad1b43b91e5929b34cf 2024-11-11T22:17:57,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4f92a30b84574f729cf05d8eb9149a78 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4f92a30b84574f729cf05d8eb9149a78 2024-11-11T22:17:57,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/63000643abe44e90b49d0f8dab0b34b1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/63000643abe44e90b49d0f8dab0b34b1 2024-11-11T22:17:57,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d71bc205ea4f49f085e6d1a8de761fc1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d71bc205ea4f49f085e6d1a8de761fc1 2024-11-11T22:17:57,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ee9909a346ae46aaa0c34a85ca679744 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ee9909a346ae46aaa0c34a85ca679744 2024-11-11T22:17:57,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3474dc378d7e46b6b8102377b6d4c6b9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3474dc378d7e46b6b8102377b6d4c6b9 2024-11-11T22:17:57,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/20c2856b199e44e2b9ee4ca546b92bf8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/20c2856b199e44e2b9ee4ca546b92bf8 2024-11-11T22:17:57,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5bdcaf87d4ef4b85b14b5faeb4916a84 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5bdcaf87d4ef4b85b14b5faeb4916a84 2024-11-11T22:17:57,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/eb5d96a991ba4d659099750344ab1e00 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/eb5d96a991ba4d659099750344ab1e00 2024-11-11T22:17:57,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ed26976aec9144ca84dd47d035b92031 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ed26976aec9144ca84dd47d035b92031 2024-11-11T22:17:57,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5dee4717845547ad929837d5572a7070 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/5dee4717845547ad929837d5572a7070 2024-11-11T22:17:57,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d20658a2e79b49d99d5acac20ea91e48 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/d20658a2e79b49d99d5acac20ea91e48 2024-11-11T22:17:57,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/80d8cbf875db422089cf320a146ace58 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/80d8cbf875db422089cf320a146ace58 2024-11-11T22:17:57,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f89e31bad0134fbab7230d9fa3da8da3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f89e31bad0134fbab7230d9fa3da8da3 2024-11-11T22:17:57,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/996ba58ec9b74a8ea37d7dce6ae6eb0b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/996ba58ec9b74a8ea37d7dce6ae6eb0b 2024-11-11T22:17:57,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/65e7e3d2340848b88d242652fdf25197 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/65e7e3d2340848b88d242652fdf25197 2024-11-11T22:17:57,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/909c34022619415897f06842e0bc37b9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/909c34022619415897f06842e0bc37b9 2024-11-11T22:17:57,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/f9d0ebc55dd54fdb8ccaac32ffd48bfc 2024-11-11T22:17:57,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/286d7b247c264be58661d9d034966c1c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/286d7b247c264be58661d9d034966c1c 2024-11-11T22:17:57,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/993e9b4b40254d738fe4dfc0687cf6a4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/993e9b4b40254d738fe4dfc0687cf6a4 2024-11-11T22:17:57,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c685facbcb0647bd867e9b62a7d0329b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c685facbcb0647bd867e9b62a7d0329b 2024-11-11T22:17:57,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ab385c7b1c234d5eb94ac39b53262a45 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/ab385c7b1c234d5eb94ac39b53262a45 2024-11-11T22:17:57,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/b015b933bed44dbfa9da8ec4b86048a7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/b015b933bed44dbfa9da8ec4b86048a7 2024-11-11T22:17:57,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3471a1caf93145d1bea000c8ca5a9950 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3471a1caf93145d1bea000c8ca5a9950 2024-11-11T22:17:57,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8784daf0f03d4e0da729d4b02fe208f3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8784daf0f03d4e0da729d4b02fe208f3 2024-11-11T22:17:57,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c90434fdbf6f43f1bfe762b992e26906 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c90434fdbf6f43f1bfe762b992e26906 2024-11-11T22:17:57,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4fa45929addb477ca05746050368e92a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4fa45929addb477ca05746050368e92a 2024-11-11T22:17:57,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4ba6079b8841402bb30a95a5d08e3b16 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/4ba6079b8841402bb30a95a5d08e3b16 2024-11-11T22:17:57,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8d5a628257714a46a7f57b10fb254938 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8d5a628257714a46a7f57b10fb254938 2024-11-11T22:17:57,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/91dc23f0ffc744fcb112f658d16c45a5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/91dc23f0ffc744fcb112f658d16c45a5 2024-11-11T22:17:57,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/884d231b789646f58783557ebc470868 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/884d231b789646f58783557ebc470868 2024-11-11T22:17:57,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/fec0bcfcc02c4dcdafb0fceaef76f096 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/fec0bcfcc02c4dcdafb0fceaef76f096 2024-11-11T22:17:57,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c56c21d2e82040e3a9c0b1f31b7cad2f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/c56c21d2e82040e3a9c0b1f31b7cad2f 2024-11-11T22:17:57,917 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/recovered.edits/527.seqid, newMaxSeqId=527, maxSeqId=1 2024-11-11T22:17:57,918 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. 2024-11-11T22:17:57,918 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for aea091a0b91074e84f623acef361ebf8: Waiting for close lock at 1731363476558Running coprocessor pre-close hooks at 1731363476558Disabling compacts and flushes for region at 1731363476558Disabling writes for close at 1731363476558Obtaining lock to block concurrent updates at 1731363476559 (+1 ms)Preparing flush snapshotting stores in aea091a0b91074e84f623acef361ebf8 at 1731363476559Finished memstore snapshotting TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731363476559Flushing stores of TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8. at 1731363476560 (+1 ms)Flushing aea091a0b91074e84f623acef361ebf8/A: creating writer at 1731363476560Flushing aea091a0b91074e84f623acef361ebf8/A: appending metadata at 1731363476562 (+2 ms)Flushing aea091a0b91074e84f623acef361ebf8/A: closing flushed file at 1731363476562Flushing aea091a0b91074e84f623acef361ebf8/B: creating writer at 1731363476970 (+408 ms)Flushing aea091a0b91074e84f623acef361ebf8/B: appending metadata at 1731363476974 (+4 ms)Flushing aea091a0b91074e84f623acef361ebf8/B: closing flushed file at 1731363476974Flushing aea091a0b91074e84f623acef361ebf8/C: creating writer at 1731363477382 (+408 ms)Flushing aea091a0b91074e84f623acef361ebf8/C: appending metadata at 1731363477384 (+2 ms)Flushing aea091a0b91074e84f623acef361ebf8/C: closing flushed file at 1731363477384Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c15840: reopening flushed file at 1731363477792 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46d39061: reopening flushed file at 1731363477796 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a81c57: reopening flushed file at 1731363477801 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for aea091a0b91074e84f623acef361ebf8 in 1248ms, sequenceid=524, compaction requested=true at 1731363477806 (+5 ms)Writing region close event to WAL at 1731363477915 (+109 ms)Running coprocessor post-close hooks at 1731363477918 (+3 ms)Closed at 1731363477918 2024-11-11T22:17:57,919 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:57,920 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=aea091a0b91074e84f623acef361ebf8, regionState=CLOSED 2024-11-11T22:17:57,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:17:57,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T22:17:57,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure aea091a0b91074e84f623acef361ebf8, server=8d82b89dd27d,36911,1731363387997 in 1.5170 sec 2024-11-11T22:17:57,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-11T22:17:57,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aea091a0b91074e84f623acef361ebf8, UNASSIGN in 1.5210 sec 2024-11-11T22:17:57,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T22:17:57,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-11T22:17:57,927 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363477927"}]},"ts":"1731363477927"} 2024-11-11T22:17:57,928 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:17:57,928 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:17:57,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5330 sec 2024-11-11T22:17:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T22:17:58,550 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:58,550 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:17:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:58,552 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:58,553 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T22:17:58,554 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:58,556 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/recovered.edits] 2024-11-11T22:17:58,558 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11829271f9d54be8b60762af8a797976 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/11829271f9d54be8b60762af8a797976 2024-11-11T22:17:58,559 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/12a756f0308b4632b7d22962c7402288 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/12a756f0308b4632b7d22962c7402288 2024-11-11T22:17:58,560 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/8fa623e79f724a74911ce7386974bbb6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/A/8fa623e79f724a74911ce7386974bbb6 2024-11-11T22:17:58,562 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/19e45a9025674d8a9d51d8a049568e66 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/19e45a9025674d8a9d51d8a049568e66 2024-11-11T22:17:58,563 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/48ed8e5b2d4146adad696e9b67d8a085 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/48ed8e5b2d4146adad696e9b67d8a085 2024-11-11T22:17:58,563 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/b1b939cb63144a0ea8bb89ffa5406ef5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/B/b1b939cb63144a0ea8bb89ffa5406ef5 2024-11-11T22:17:58,565 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3b8a9d1d8cd9409280956c3636ee2f11 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/3b8a9d1d8cd9409280956c3636ee2f11 2024-11-11T22:17:58,565 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8da746cc4ad84b1b9e23d2694b60cf72 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/8da746cc4ad84b1b9e23d2694b60cf72 2024-11-11T22:17:58,566 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/cd212ca0d7934dd8b8d73954338ceca9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/C/cd212ca0d7934dd8b8d73954338ceca9 2024-11-11T22:17:58,568 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/recovered.edits/527.seqid to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8/recovered.edits/527.seqid 2024-11-11T22:17:58,569 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/aea091a0b91074e84f623acef361ebf8 2024-11-11T22:17:58,569 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:17:58,571 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:58,574 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:17:58,576 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:17:58,577 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:58,577 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:17:58,577 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731363478577"}]},"ts":"9223372036854775807"} 2024-11-11T22:17:58,579 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:17:58,579 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aea091a0b91074e84f623acef361ebf8, NAME => 'TestAcidGuarantees,,1731363450906.aea091a0b91074e84f623acef361ebf8.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:17:58,579 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:17:58,579 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731363478579"}]},"ts":"9223372036854775807"} 2024-11-11T22:17:58,580 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:17:58,581 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:58,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-11T22:17:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T22:17:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:17:58,663 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:17:58,669 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251 (was 251), OpenFileDescriptor=457 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=784 (was 780) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1011 (was 1932) 2024-11-11T22:17:58,678 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=784, ProcessCount=11, AvailableMemoryMB=1011 2024-11-11T22:17:58,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:17:58,679 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:17:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:17:58,682 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:17:58,682 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:58,682 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-11T22:17:58,683 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:17:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T22:17:58,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742161_1337 (size=1061) 2024-11-11T22:17:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T22:17:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T22:17:59,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e 2024-11-11T22:17:59,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742162_1338 (size=53) 2024-11-11T22:17:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T22:17:59,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:17:59,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 76df2be919cd8ff2500e96d76dee493a, disabling compactions & flushes 2024-11-11T22:17:59,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:17:59,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:17:59,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. after waiting 0 ms 2024-11-11T22:17:59,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:17:59,499 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:17:59,499 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76df2be919cd8ff2500e96d76dee493a: Waiting for close lock at 1731363479499Disabling compacts and flushes for region at 1731363479499Disabling writes for close at 1731363479499Writing region close event to WAL at 1731363479499Closed at 1731363479499 2024-11-11T22:17:59,500 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:17:59,500 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363479500"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363479500"}]},"ts":"1731363479500"} 2024-11-11T22:17:59,501 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:17:59,502 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:17:59,502 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363479502"}]},"ts":"1731363479502"} 2024-11-11T22:17:59,503 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:17:59,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, ASSIGN}] 2024-11-11T22:17:59,504 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, ASSIGN 2024-11-11T22:17:59,505 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, ASSIGN; state=OFFLINE, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=false 2024-11-11T22:17:59,656 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=76df2be919cd8ff2500e96d76dee493a, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:59,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, ASSIGN because future has completed 2024-11-11T22:17:59,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:17:59,819 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:17:59,820 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:17:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T22:17:59,820 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,820 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:17:59,821 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,821 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,824 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,827 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:59,827 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76df2be919cd8ff2500e96d76dee493a columnFamilyName A 2024-11-11T22:17:59,827 DEBUG [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:59,828 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(327): Store=76df2be919cd8ff2500e96d76dee493a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:59,828 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,830 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:59,830 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76df2be919cd8ff2500e96d76dee493a columnFamilyName B 2024-11-11T22:17:59,830 DEBUG [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:59,830 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(327): Store=76df2be919cd8ff2500e96d76dee493a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:59,831 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,832 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:17:59,832 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76df2be919cd8ff2500e96d76dee493a columnFamilyName C 2024-11-11T22:17:59,832 DEBUG [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:17:59,832 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(327): Store=76df2be919cd8ff2500e96d76dee493a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:17:59,833 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:17:59,833 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,834 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,834 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,835 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,835 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,836 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:17:59,837 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,839 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:17:59,840 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 76df2be919cd8ff2500e96d76dee493a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60231580, jitterRate=-0.10247951745986938}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:17:59,840 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:17:59,841 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 76df2be919cd8ff2500e96d76dee493a: Running coprocessor pre-open hook at 1731363479821Writing region info on filesystem at 1731363479821Initializing all the Stores at 1731363479823 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363479823Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363479823Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363479823Cleaning up temporary data from old regions at 1731363479835 (+12 ms)Running coprocessor post-open hooks at 1731363479840 (+5 ms)Region opened successfully at 1731363479841 (+1 ms) 2024-11-11T22:17:59,842 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., pid=89, masterSystemTime=1731363479814 2024-11-11T22:17:59,844 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:17:59,844 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:17:59,844 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=76df2be919cd8ff2500e96d76dee493a, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:17:59,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:17:59,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T22:17:59,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 in 186 msec 2024-11-11T22:17:59,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T22:17:59,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, ASSIGN in 346 msec 2024-11-11T22:17:59,853 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:17:59,853 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363479853"}]},"ts":"1731363479853"} 2024-11-11T22:17:59,854 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:17:59,855 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:17:59,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-11T22:18:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T22:18:00,831 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:00,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11040955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:00,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:00,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:00,835 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bb6242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:00,838 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32147737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:00,839 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:00,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:00,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:18:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,41919,1731363387169 2024-11-11T22:18:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64d72008 2024-11-11T22:18:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:18:00,845 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:18:00,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:18:00,847 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:18:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:00,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742163_1339 (size=1097) 2024-11-11T22:18:01,261 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T22:18:01,261 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T22:18:01,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:18:01,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, REOPEN/MOVE}] 2024-11-11T22:18:01,267 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, REOPEN/MOVE 2024-11-11T22:18:01,267 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=76df2be919cd8ff2500e96d76dee493a, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:01,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, REOPEN/MOVE because future has completed 2024-11-11T22:18:01,268 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:18:01,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:18:01,421 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,421 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:18:01,421 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 76df2be919cd8ff2500e96d76dee493a, disabling compactions & flushes 2024-11-11T22:18:01,421 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,421 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,421 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. after waiting 0 ms 2024-11-11T22:18:01,421 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,425 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T22:18:01,425 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,425 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 76df2be919cd8ff2500e96d76dee493a: Waiting for close lock at 1731363481421Running coprocessor pre-close hooks at 1731363481421Disabling compacts and flushes for region at 1731363481421Disabling writes for close at 1731363481421Writing region close event to WAL at 1731363481422 (+1 ms)Running coprocessor post-close hooks at 1731363481425 (+3 ms)Closed at 1731363481425 2024-11-11T22:18:01,425 WARN [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 76df2be919cd8ff2500e96d76dee493a to self. 2024-11-11T22:18:01,427 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,428 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=76df2be919cd8ff2500e96d76dee493a, regionState=CLOSED 2024-11-11T22:18:01,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:18:01,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-11T22:18:01,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 in 161 msec 2024-11-11T22:18:01,432 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, REOPEN/MOVE; state=CLOSED, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=true 2024-11-11T22:18:01,582 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=76df2be919cd8ff2500e96d76dee493a, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:01,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, REOPEN/MOVE because future has completed 2024-11-11T22:18:01,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:18:01,739 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,739 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:18:01,740 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,740 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:18:01,740 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,740 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,742 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,743 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:01,743 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76df2be919cd8ff2500e96d76dee493a columnFamilyName A 2024-11-11T22:18:01,745 DEBUG [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:01,745 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(327): Store=76df2be919cd8ff2500e96d76dee493a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:01,746 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,747 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:01,747 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76df2be919cd8ff2500e96d76dee493a columnFamilyName B 2024-11-11T22:18:01,747 DEBUG [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:01,748 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(327): Store=76df2be919cd8ff2500e96d76dee493a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:01,748 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,748 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:01,748 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76df2be919cd8ff2500e96d76dee493a columnFamilyName C 2024-11-11T22:18:01,748 DEBUG [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:01,749 INFO [StoreOpener-76df2be919cd8ff2500e96d76dee493a-1 {}] regionserver.HStore(327): Store=76df2be919cd8ff2500e96d76dee493a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:01,749 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,749 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,749 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,750 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,751 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,751 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,751 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:18:01,752 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,753 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 76df2be919cd8ff2500e96d76dee493a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73451066, jitterRate=0.094506174325943}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:18:01,753 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,753 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 76df2be919cd8ff2500e96d76dee493a: Running coprocessor pre-open hook at 1731363481740Writing region info on filesystem at 1731363481740Initializing all the Stores at 1731363481741 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363481741Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363481741Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363481741Cleaning up temporary data from old regions at 1731363481751 (+10 ms)Running coprocessor post-open hooks at 1731363481753 (+2 ms)Region opened successfully at 1731363481753 2024-11-11T22:18:01,754 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., pid=94, masterSystemTime=1731363481736 2024-11-11T22:18:01,755 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,755 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,756 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=76df2be919cd8ff2500e96d76dee493a, regionState=OPEN, openSeqNum=5, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:01,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:18:01,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-11T22:18:01,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 in 173 msec 2024-11-11T22:18:01,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T22:18:01,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, REOPEN/MOVE in 493 msec 2024-11-11T22:18:01,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-11T22:18:01,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-11-11T22:18:01,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-11T22:18:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T22:18:01,766 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:01,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15622d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,768 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edae12d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,770 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a8f469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25061205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,771 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7216da21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68bfdb62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d28306f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61736877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,777 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fbbc119, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29412344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,778 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24501eb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,779 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@795f7aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6365488e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,781 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4443e1c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,782 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c0304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5109bfd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,784 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54128623, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,785 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f243986, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563dd2d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,786 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0afd17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,788 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5669674e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c93281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,789 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c85a7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,790 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134451d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5938121e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,792 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223f4f34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,793 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fcbdf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40993d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:01,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:01,795 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb7cbef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:01,796 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd91ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:01,797 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,797 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,797 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,797 DEBUG [Thread-1543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,798 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,798 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,799 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T22:18:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,800 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:01,800 DEBUG [Thread-1556 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:18:01,800 DEBUG [Thread-1556 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,801 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,801 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:01,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:01,801 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,802 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:01,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:18:01,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:01,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:01,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:01,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:01,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:01,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:01,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111827195a2bf954e6c900c74df36ef88fa_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363481812/Put/seqid=0 2024-11-11T22:18:01,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363481855, max timestamp=1731363481874Num uniques 859; 2024-11-11T22:18:01,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:01,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742164_1340 (size=12154) 2024-11-11T22:18:01,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363481855, max timestamp=1731363481874Num uniques 859; 2024-11-11T22:18:01,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363481855, max timestamp=1731363481875Num uniques 859; 2024-11-11T22:18:01,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:01,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:01,882 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:01,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111827195a2bf954e6c900c74df36ef88fa_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111827195a2bf954e6c900c74df36ef88fa_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:01,894 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/27c85e0d3fb44f2d878bf1d33175b10e, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:01,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/27c85e0d3fb44f2d878bf1d33175b10e is 175, key is test_row_0/A:col10/1731363481812/Put/seqid=0 2024-11-11T22:18:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:18:01,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742165_1341 (size=30951) 2024-11-11T22:18:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363541918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:01,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363541922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363541922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363541924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363541925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:01,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:01,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:01,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:01,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363542045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363542050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363542051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363542051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363542051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:18:02,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363542270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363542270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363542270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363542271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363542271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/27c85e0d3fb44f2d878bf1d33175b10e 2024-11-11T22:18:02,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/d4c8349d1d054287a31cbc9b59bc2f4c is 50, key is test_row_0/B:col10/1731363481812/Put/seqid=0 2024-11-11T22:18:02,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742166_1342 (size=9657) 2024-11-11T22:18:02,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:02,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:02,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:18:02,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:02,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:02,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363542589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363542589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363542590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363542590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363542590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:02,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:02,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:02,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/d4c8349d1d054287a31cbc9b59bc2f4c 2024-11-11T22:18:02,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/ee107ab4a20044b49fac9fef164e9e59 is 50, key is test_row_0/C:col10/1731363481812/Put/seqid=0 2024-11-11T22:18:02,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742167_1343 (size=9657) 2024-11-11T22:18:02,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:02,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:18:03,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:03,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:03,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:03,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363543099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363543099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363543099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363543099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363543103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:03,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:03,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/ee107ab4a20044b49fac9fef164e9e59 2024-11-11T22:18:03,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:03,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:03,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/27c85e0d3fb44f2d878bf1d33175b10e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27c85e0d3fb44f2d878bf1d33175b10e 2024-11-11T22:18:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:03,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27c85e0d3fb44f2d878bf1d33175b10e, entries=150, sequenceid=14, filesize=30.2 K 2024-11-11T22:18:03,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/d4c8349d1d054287a31cbc9b59bc2f4c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d4c8349d1d054287a31cbc9b59bc2f4c 2024-11-11T22:18:03,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d4c8349d1d054287a31cbc9b59bc2f4c, entries=100, sequenceid=14, filesize=9.4 K 2024-11-11T22:18:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/ee107ab4a20044b49fac9fef164e9e59 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/ee107ab4a20044b49fac9fef164e9e59 2024-11-11T22:18:03,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/ee107ab4a20044b49fac9fef164e9e59, entries=100, sequenceid=14, filesize=9.4 K 2024-11-11T22:18:03,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 76df2be919cd8ff2500e96d76dee493a in 1388ms, sequenceid=14, compaction requested=false 2024-11-11T22:18:03,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:03,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T22:18:03,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:03,340 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T22:18:03,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:03,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:03,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:03,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:03,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:03,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:03,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111117cc2dd20f6f4b05a3ba5ae3af9e19b5_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363481923/Put/seqid=0 2024-11-11T22:18:03,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742168_1344 (size=12154) 2024-11-11T22:18:03,447 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:18:03,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:03,761 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111117cc2dd20f6f4b05a3ba5ae3af9e19b5_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117cc2dd20f6f4b05a3ba5ae3af9e19b5_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1bf5cd6d0fa24e32b7c24402e441f596, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1bf5cd6d0fa24e32b7c24402e441f596 is 175, key is test_row_0/A:col10/1731363481923/Put/seqid=0 2024-11-11T22:18:03,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742169_1345 (size=30955) 2024-11-11T22:18:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:18:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:04,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:04,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363544126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363544129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363544130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363544131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,136 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 76df2be919cd8ff2500e96d76dee493a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363544133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,167 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1bf5cd6d0fa24e32b7c24402e441f596 2024-11-11T22:18:04,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/1d4a7efa2e2c4694871a5273bdf2fb23 is 50, key is test_row_0/B:col10/1731363481923/Put/seqid=0 2024-11-11T22:18:04,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742170_1346 (size=12001) 2024-11-11T22:18:04,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363544239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363544240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363544240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363544249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363544449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363544450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363544450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363544469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,577 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/1d4a7efa2e2c4694871a5273bdf2fb23 2024-11-11T22:18:04,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/b2b678e3b85d480095ab6f185e4e005b is 50, key is test_row_0/C:col10/1731363481923/Put/seqid=0 2024-11-11T22:18:04,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742171_1347 (size=12001) 2024-11-11T22:18:04,604 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/b2b678e3b85d480095ab6f185e4e005b 2024-11-11T22:18:04,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1bf5cd6d0fa24e32b7c24402e441f596 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1bf5cd6d0fa24e32b7c24402e441f596 2024-11-11T22:18:04,614 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1bf5cd6d0fa24e32b7c24402e441f596, entries=150, sequenceid=50, filesize=30.2 K 2024-11-11T22:18:04,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/1d4a7efa2e2c4694871a5273bdf2fb23 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1d4a7efa2e2c4694871a5273bdf2fb23 2024-11-11T22:18:04,621 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1d4a7efa2e2c4694871a5273bdf2fb23, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T22:18:04,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/b2b678e3b85d480095ab6f185e4e005b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2b678e3b85d480095ab6f185e4e005b 2024-11-11T22:18:04,626 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2b678e3b85d480095ab6f185e4e005b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T22:18:04,627 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 76df2be919cd8ff2500e96d76dee493a in 1287ms, sequenceid=50, compaction requested=false 2024-11-11T22:18:04,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:04,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:04,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T22:18:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T22:18:04,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T22:18:04,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8280 sec 2024-11-11T22:18:04,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.8340 sec 2024-11-11T22:18:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:04,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T22:18:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:04,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146babdb55db24554985743e11ec2ac6b_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363484765/Put/seqid=0 2024-11-11T22:18:04,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742172_1348 (size=12154) 2024-11-11T22:18:04,798 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:04,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146babdb55db24554985743e11ec2ac6b_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146babdb55db24554985743e11ec2ac6b_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:04,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/4497f51b77b645ff959bc6f737d7c662, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:04,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/4497f51b77b645ff959bc6f737d7c662 is 175, key is test_row_0/A:col10/1731363484765/Put/seqid=0 2024-11-11T22:18:04,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742173_1349 (size=30955) 2024-11-11T22:18:04,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/4497f51b77b645ff959bc6f737d7c662 2024-11-11T22:18:04,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/86dbd244b11945dd9b8982bb9b013fe3 is 50, key is test_row_0/B:col10/1731363484765/Put/seqid=0 2024-11-11T22:18:04,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742174_1350 (size=12001) 2024-11-11T22:18:04,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363544817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363544817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363544829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363544829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363544939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363544939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363544949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363544949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363545150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363545150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363545159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363545169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/86dbd244b11945dd9b8982bb9b013fe3 2024-11-11T22:18:05,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/9aec473cf43b4fde807fb86bf6e92e36 is 50, key is test_row_0/C:col10/1731363484765/Put/seqid=0 2024-11-11T22:18:05,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742175_1351 (size=12001) 2024-11-11T22:18:05,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/9aec473cf43b4fde807fb86bf6e92e36 2024-11-11T22:18:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/4497f51b77b645ff959bc6f737d7c662 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4497f51b77b645ff959bc6f737d7c662 2024-11-11T22:18:05,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4497f51b77b645ff959bc6f737d7c662, entries=150, sequenceid=62, filesize=30.2 K 2024-11-11T22:18:05,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/86dbd244b11945dd9b8982bb9b013fe3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/86dbd244b11945dd9b8982bb9b013fe3 2024-11-11T22:18:05,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/86dbd244b11945dd9b8982bb9b013fe3, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T22:18:05,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/9aec473cf43b4fde807fb86bf6e92e36 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/9aec473cf43b4fde807fb86bf6e92e36 2024-11-11T22:18:05,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/9aec473cf43b4fde807fb86bf6e92e36, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T22:18:05,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 76df2be919cd8ff2500e96d76dee493a in 533ms, sequenceid=62, compaction requested=true 2024-11-11T22:18:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:05,299 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:05,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:05,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:05,300 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:05,301 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:05,301 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/A is initiating minor compaction (all files) 2024-11-11T22:18:05,301 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/A in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:05,301 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27c85e0d3fb44f2d878bf1d33175b10e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1bf5cd6d0fa24e32b7c24402e441f596, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4497f51b77b645ff959bc6f737d7c662] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=90.7 K 2024-11-11T22:18:05,301 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:05,301 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27c85e0d3fb44f2d878bf1d33175b10e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1bf5cd6d0fa24e32b7c24402e441f596, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4497f51b77b645ff959bc6f737d7c662] 2024-11-11T22:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:05,301 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:05,301 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/B is initiating minor compaction (all files) 2024-11-11T22:18:05,301 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/B in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:05,301 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d4c8349d1d054287a31cbc9b59bc2f4c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1d4a7efa2e2c4694871a5273bdf2fb23, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/86dbd244b11945dd9b8982bb9b013fe3] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=32.9 K 2024-11-11T22:18:05,301 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c85e0d3fb44f2d878bf1d33175b10e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363481810 2024-11-11T22:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:05,302 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c8349d1d054287a31cbc9b59bc2f4c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363481811 2024-11-11T22:18:05,302 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bf5cd6d0fa24e32b7c24402e441f596, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363481905 2024-11-11T22:18:05,302 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4a7efa2e2c4694871a5273bdf2fb23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363481905 2024-11-11T22:18:05,302 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4497f51b77b645ff959bc6f737d7c662, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363484125 2024-11-11T22:18:05,303 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 86dbd244b11945dd9b8982bb9b013fe3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363484125 2024-11-11T22:18:05,319 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:05,320 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/726282ed02514730bdebc66ac4c15473 is 50, key is test_row_0/B:col10/1731363484765/Put/seqid=0 2024-11-11T22:18:05,329 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:05,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742176_1352 (size=12104) 2024-11-11T22:18:05,346 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110d72acfee25245ed923a6570ffe24e70_76df2be919cd8ff2500e96d76dee493a store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:05,348 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110d72acfee25245ed923a6570ffe24e70_76df2be919cd8ff2500e96d76dee493a, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:05,348 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110d72acfee25245ed923a6570ffe24e70_76df2be919cd8ff2500e96d76dee493a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:05,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742177_1353 (size=4469) 2024-11-11T22:18:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:05,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T22:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:05,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110f3f9fe275e4ddba2c64243a8478ede_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363485473/Put/seqid=0 2024-11-11T22:18:05,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742178_1354 (size=14594) 2024-11-11T22:18:05,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:05,488 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110f3f9fe275e4ddba2c64243a8478ede_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110f3f9fe275e4ddba2c64243a8478ede_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:05,489 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/11bc83e228834e7faa015b80bdd95841, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:05,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/11bc83e228834e7faa015b80bdd95841 is 175, key is test_row_0/A:col10/1731363485473/Put/seqid=0 2024-11-11T22:18:05,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742179_1355 (size=39549) 2024-11-11T22:18:05,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363545491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363545493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363545494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363545505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363545609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363545609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363545610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363545619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,745 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/726282ed02514730bdebc66ac4c15473 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/726282ed02514730bdebc66ac4c15473 2024-11-11T22:18:05,752 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/B of 76df2be919cd8ff2500e96d76dee493a into 726282ed02514730bdebc66ac4c15473(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:05,752 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:05,752 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/B, priority=13, startTime=1731363485300; duration=0sec 2024-11-11T22:18:05,752 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:05,752 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:B 2024-11-11T22:18:05,752 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:05,753 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:05,753 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/C is initiating minor compaction (all files) 2024-11-11T22:18:05,753 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/C in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:05,753 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/ee107ab4a20044b49fac9fef164e9e59, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2b678e3b85d480095ab6f185e4e005b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/9aec473cf43b4fde807fb86bf6e92e36] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=32.9 K 2024-11-11T22:18:05,753 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ee107ab4a20044b49fac9fef164e9e59, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363481811 2024-11-11T22:18:05,754 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b2b678e3b85d480095ab6f185e4e005b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731363481905 2024-11-11T22:18:05,754 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aec473cf43b4fde807fb86bf6e92e36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363484125 2024-11-11T22:18:05,755 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:05,755 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/12340b8e65d24068b62d1dfcbf92058f is 175, key is test_row_0/A:col10/1731363484765/Put/seqid=0 2024-11-11T22:18:05,768 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#C#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:05,768 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/853af3561f304e3f89a7a1406719a5d0 is 50, key is test_row_0/C:col10/1731363484765/Put/seqid=0 2024-11-11T22:18:05,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742181_1357 (size=12104) 2024-11-11T22:18:05,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742180_1356 (size=31058) 2024-11-11T22:18:05,781 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/853af3561f304e3f89a7a1406719a5d0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/853af3561f304e3f89a7a1406719a5d0 2024-11-11T22:18:05,788 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/C of 76df2be919cd8ff2500e96d76dee493a into 853af3561f304e3f89a7a1406719a5d0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:05,788 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:05,788 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/C, priority=13, startTime=1731363485301; duration=0sec 2024-11-11T22:18:05,789 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:05,789 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:C 2024-11-11T22:18:05,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363545829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363545830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363545830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363545830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:05,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/11bc83e228834e7faa015b80bdd95841 2024-11-11T22:18:05,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/1fb9a2126e5146b1898566174ef23f91 is 50, key is test_row_0/B:col10/1731363485473/Put/seqid=0 2024-11-11T22:18:05,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742182_1358 (size=12001) 2024-11-11T22:18:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T22:18:05,959 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:05,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T22:18:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:18:05,961 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:05,962 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:05,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:18:06,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:18:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363546149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363546149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363546149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,153 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 76df2be919cd8ff2500e96d76dee493a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:18:06,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363546149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363546149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,179 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/12340b8e65d24068b62d1dfcbf92058f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/12340b8e65d24068b62d1dfcbf92058f 2024-11-11T22:18:06,183 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/A of 76df2be919cd8ff2500e96d76dee493a into 12340b8e65d24068b62d1dfcbf92058f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:06,183 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:06,183 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/A, priority=13, startTime=1731363485299; duration=0sec 2024-11-11T22:18:06,183 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:06,183 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:A 2024-11-11T22:18:06,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:18:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:18:06,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/1fb9a2126e5146b1898566174ef23f91 2024-11-11T22:18:06,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/035d50318d954df8b41133fd32c5f462 is 50, key is test_row_0/C:col10/1731363485473/Put/seqid=0 2024-11-11T22:18:06,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742183_1359 (size=12001) 2024-11-11T22:18:06,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:18:06,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:06,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:18:06,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:06,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:18:06,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363546659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363546659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363546660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363546660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:06,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:18:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:06,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/035d50318d954df8b41133fd32c5f462 2024-11-11T22:18:06,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/11bc83e228834e7faa015b80bdd95841 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/11bc83e228834e7faa015b80bdd95841 2024-11-11T22:18:06,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/11bc83e228834e7faa015b80bdd95841, entries=200, sequenceid=83, filesize=38.6 K 2024-11-11T22:18:06,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/1fb9a2126e5146b1898566174ef23f91 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1fb9a2126e5146b1898566174ef23f91 2024-11-11T22:18:06,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1fb9a2126e5146b1898566174ef23f91, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T22:18:06,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/035d50318d954df8b41133fd32c5f462 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/035d50318d954df8b41133fd32c5f462 2024-11-11T22:18:06,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/035d50318d954df8b41133fd32c5f462, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T22:18:06,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 76df2be919cd8ff2500e96d76dee493a in 1294ms, sequenceid=83, compaction requested=false 2024-11-11T22:18:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:06,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T22:18:06,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:06,884 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T22:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:06,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115a40640eab824a12965051adefa6bd03_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363485493/Put/seqid=0 2024-11-11T22:18:06,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742184_1360 (size=12154) 2024-11-11T22:18:06,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:06,912 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115a40640eab824a12965051adefa6bd03_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a40640eab824a12965051adefa6bd03_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:06,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ede11b12a8e14690a9e401479195fd3a, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:06,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ede11b12a8e14690a9e401479195fd3a is 175, key is test_row_0/A:col10/1731363485493/Put/seqid=0 2024-11-11T22:18:06,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742185_1361 (size=30955) 2024-11-11T22:18:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:18:07,333 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ede11b12a8e14690a9e401479195fd3a 2024-11-11T22:18:07,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/ca659c9eb90b40a2a62cc8250c7605ec is 50, key is test_row_0/B:col10/1731363485493/Put/seqid=0 2024-11-11T22:18:07,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742186_1362 (size=12001) 2024-11-11T22:18:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:07,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:07,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:07,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363547725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:07,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363547727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:07,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:07,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363547729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:07,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363487722, max timestamp=1731363487739Num uniques 859; 2024-11-11T22:18:07,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363487722, max timestamp=1731363487739Num uniques 859; 2024-11-11T22:18:07,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:18:07,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:18:07,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:07,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363487722, max timestamp=1731363487739Num uniques 859; 2024-11-11T22:18:07,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:18:07,756 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/ca659c9eb90b40a2a62cc8250c7605ec 2024-11-11T22:18:07,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/278b037d154941c5a1228e46ff76cb28 is 50, key is test_row_0/C:col10/1731363485493/Put/seqid=0 2024-11-11T22:18:07,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742187_1363 (size=12001) 2024-11-11T22:18:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363547866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:07,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363547867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:07,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363547876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:07,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363547878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363547989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363547989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363547989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363547999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:18:08,186 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/278b037d154941c5a1228e46ff76cb28 2024-11-11T22:18:08,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ede11b12a8e14690a9e401479195fd3a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ede11b12a8e14690a9e401479195fd3a 2024-11-11T22:18:08,194 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ede11b12a8e14690a9e401479195fd3a, entries=150, sequenceid=97, filesize=30.2 K 2024-11-11T22:18:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/ca659c9eb90b40a2a62cc8250c7605ec as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ca659c9eb90b40a2a62cc8250c7605ec 2024-11-11T22:18:08,198 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ca659c9eb90b40a2a62cc8250c7605ec, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T22:18:08,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/278b037d154941c5a1228e46ff76cb28 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/278b037d154941c5a1228e46ff76cb28 2024-11-11T22:18:08,208 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/278b037d154941c5a1228e46ff76cb28, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T22:18:08,209 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76df2be919cd8ff2500e96d76dee493a in 1325ms, sequenceid=97, compaction requested=true 2024-11-11T22:18:08,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:08,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:08,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T22:18:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T22:18:08,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T22:18:08,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2490 sec 2024-11-11T22:18:08,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.2530 sec 2024-11-11T22:18:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:08,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T22:18:08,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:08,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:08,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:08,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:08,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:08,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:08,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111138cd8b73bb7f4eb68dac1dd76437d003_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363487860/Put/seqid=0 2024-11-11T22:18:08,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742188_1364 (size=14744) 2024-11-11T22:18:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363548230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363548236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363548240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363548241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363548349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363548349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363548359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363548360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363548569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363548569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363548570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363548570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:08,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111138cd8b73bb7f4eb68dac1dd76437d003_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111138cd8b73bb7f4eb68dac1dd76437d003_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:08,641 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/3d0abc79c2974d808e593abfc2458402, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:08,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/3d0abc79c2974d808e593abfc2458402 is 175, key is test_row_0/A:col10/1731363487860/Put/seqid=0 2024-11-11T22:18:08,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742189_1365 (size=39699) 2024-11-11T22:18:08,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/3d0abc79c2974d808e593abfc2458402 2024-11-11T22:18:08,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/15d343c6cd2545cda70c8c01d26e9d35 is 50, key is test_row_0/B:col10/1731363487860/Put/seqid=0 2024-11-11T22:18:08,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742190_1366 (size=12101) 2024-11-11T22:18:08,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/15d343c6cd2545cda70c8c01d26e9d35 2024-11-11T22:18:08,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/fc00db0920ce4fcdbe5acce32c65c5c5 is 50, key is test_row_0/C:col10/1731363487860/Put/seqid=0 2024-11-11T22:18:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742191_1367 (size=12101) 2024-11-11T22:18:08,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363548899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363548899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363548899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363548899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/fc00db0920ce4fcdbe5acce32c65c5c5 2024-11-11T22:18:09,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/3d0abc79c2974d808e593abfc2458402 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3d0abc79c2974d808e593abfc2458402 2024-11-11T22:18:09,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3d0abc79c2974d808e593abfc2458402, entries=200, sequenceid=134, filesize=38.8 K 2024-11-11T22:18:09,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/15d343c6cd2545cda70c8c01d26e9d35 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/15d343c6cd2545cda70c8c01d26e9d35 2024-11-11T22:18:09,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/15d343c6cd2545cda70c8c01d26e9d35, entries=150, sequenceid=134, filesize=11.8 K 2024-11-11T22:18:09,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/fc00db0920ce4fcdbe5acce32c65c5c5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fc00db0920ce4fcdbe5acce32c65c5c5 2024-11-11T22:18:09,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fc00db0920ce4fcdbe5acce32c65c5c5, entries=150, sequenceid=134, filesize=11.8 K 2024-11-11T22:18:09,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 76df2be919cd8ff2500e96d76dee493a in 895ms, sequenceid=134, compaction requested=true 2024-11-11T22:18:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:09,116 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:18:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:09,116 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:18:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:09,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:09,117 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141261 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:18:09,117 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:18:09,117 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/A is initiating minor compaction (all files) 2024-11-11T22:18:09,117 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/B is initiating minor compaction (all files) 2024-11-11T22:18:09,117 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/A in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:09,117 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/B in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:09,117 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/12340b8e65d24068b62d1dfcbf92058f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/11bc83e228834e7faa015b80bdd95841, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ede11b12a8e14690a9e401479195fd3a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3d0abc79c2974d808e593abfc2458402] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=138.0 K 2024-11-11T22:18:09,117 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/726282ed02514730bdebc66ac4c15473, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1fb9a2126e5146b1898566174ef23f91, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ca659c9eb90b40a2a62cc8250c7605ec, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/15d343c6cd2545cda70c8c01d26e9d35] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=47.1 K 2024-11-11T22:18:09,118 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/12340b8e65d24068b62d1dfcbf92058f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/11bc83e228834e7faa015b80bdd95841, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ede11b12a8e14690a9e401479195fd3a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3d0abc79c2974d808e593abfc2458402] 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12340b8e65d24068b62d1dfcbf92058f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363484125 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 726282ed02514730bdebc66ac4c15473, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363484125 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11bc83e228834e7faa015b80bdd95841, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731363484815 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb9a2126e5146b1898566174ef23f91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731363484815 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede11b12a8e14690a9e401479195fd3a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363485492 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ca659c9eb90b40a2a62cc8250c7605ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363485492 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d0abc79c2974d808e593abfc2458402, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363487860 2024-11-11T22:18:09,118 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 15d343c6cd2545cda70c8c01d26e9d35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363487860 2024-11-11T22:18:09,128 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#B#compaction#315 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:09,128 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/3321b41534aa4fd1a128a76a69504554 is 50, key is test_row_0/B:col10/1731363487860/Put/seqid=0 2024-11-11T22:18:09,128 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:09,133 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118fa0ae0ab5b34b8788f9c56685fb21a7_76df2be919cd8ff2500e96d76dee493a store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:09,135 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118fa0ae0ab5b34b8788f9c56685fb21a7_76df2be919cd8ff2500e96d76dee493a, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:09,136 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118fa0ae0ab5b34b8788f9c56685fb21a7_76df2be919cd8ff2500e96d76dee493a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:09,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742192_1368 (size=12341) 2024-11-11T22:18:09,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742193_1369 (size=4469) 2024-11-11T22:18:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:09,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T22:18:09,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:09,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:09,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:09,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:09,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:09,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:09,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111d50bc38e31a4ac6b26b5122d775a17e_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363489426/Put/seqid=0 2024-11-11T22:18:09,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742194_1370 (size=14794) 2024-11-11T22:18:09,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731363489447, max timestamp=1731363489485Num uniques 859; 2024-11-11T22:18:09,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:18:09,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731363489447, max timestamp=1731363489485Num uniques 859; 2024-11-11T22:18:09,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:18:09,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731363489447, max timestamp=1731363489485Num uniques 859; 2024-11-11T22:18:09,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:18:09,544 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/3321b41534aa4fd1a128a76a69504554 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3321b41534aa4fd1a128a76a69504554 2024-11-11T22:18:09,547 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#A#compaction#316 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:09,547 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ba8d322c097246c28cb63bcf80df993e is 175, key is test_row_0/A:col10/1731363487860/Put/seqid=0 2024-11-11T22:18:09,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363549526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363549527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742195_1371 (size=31295) 2024-11-11T22:18:09,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363549542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,552 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/B of 76df2be919cd8ff2500e96d76dee493a into 3321b41534aa4fd1a128a76a69504554(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:09,552 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:09,552 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/B, priority=12, startTime=1731363489116; duration=0sec 2024-11-11T22:18:09,552 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:09,552 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:B 2024-11-11T22:18:09,552 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:18:09,553 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:18:09,553 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/C is initiating minor compaction (all files) 2024-11-11T22:18:09,553 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/C in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:09,553 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/853af3561f304e3f89a7a1406719a5d0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/035d50318d954df8b41133fd32c5f462, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/278b037d154941c5a1228e46ff76cb28, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fc00db0920ce4fcdbe5acce32c65c5c5] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=47.1 K 2024-11-11T22:18:09,554 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 853af3561f304e3f89a7a1406719a5d0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731363484125 2024-11-11T22:18:09,554 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 035d50318d954df8b41133fd32c5f462, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731363484815 2024-11-11T22:18:09,555 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 278b037d154941c5a1228e46ff76cb28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363485492 2024-11-11T22:18:09,555 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fc00db0920ce4fcdbe5acce32c65c5c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363487860 2024-11-11T22:18:09,555 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ba8d322c097246c28cb63bcf80df993e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ba8d322c097246c28cb63bcf80df993e 2024-11-11T22:18:09,560 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/A of 76df2be919cd8ff2500e96d76dee493a into ba8d322c097246c28cb63bcf80df993e(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:09,560 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:09,560 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/A, priority=12, startTime=1731363489116; duration=0sec 2024-11-11T22:18:09,560 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:09,560 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:A 2024-11-11T22:18:09,562 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:09,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,562 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/12c45f8de1fe443c8dd7c36ac285584c is 50, key is test_row_0/C:col10/1731363487860/Put/seqid=0 2024-11-11T22:18:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363549550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742196_1372 (size=12341) 2024-11-11T22:18:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363549659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363549659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363549659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363549669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:09,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111d50bc38e31a4ac6b26b5122d775a17e_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111d50bc38e31a4ac6b26b5122d775a17e_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:09,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/67c3f235087e411cb3b5052a87fc23b7, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:09,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/67c3f235087e411cb3b5052a87fc23b7 is 175, key is test_row_0/A:col10/1731363489426/Put/seqid=0 2024-11-11T22:18:09,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742197_1373 (size=39749) 2024-11-11T22:18:09,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363549879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363549879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363549879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363549879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:09,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:09,978 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/12c45f8de1fe443c8dd7c36ac285584c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/12c45f8de1fe443c8dd7c36ac285584c 2024-11-11T22:18:09,982 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/C of 76df2be919cd8ff2500e96d76dee493a into 12c45f8de1fe443c8dd7c36ac285584c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:09,982 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:09,982 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/C, priority=12, startTime=1731363489116; duration=0sec 2024-11-11T22:18:09,982 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:09,982 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:C 2024-11-11T22:18:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T22:18:10,119 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:10,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T22:18:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:18:10,121 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:10,122 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:10,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:10,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363550160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,166 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in 76df2be919cd8ff2500e96d76dee493a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:18:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363550199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363550199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363550200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363550200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:18:10,254 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/67c3f235087e411cb3b5052a87fc23b7 2024-11-11T22:18:10,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/a40b1eacb15f4c5482105ac0c05850b1 is 50, key is test_row_0/B:col10/1731363489426/Put/seqid=0 2024-11-11T22:18:10,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742198_1374 (size=12151) 2024-11-11T22:18:10,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:18:10,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/a40b1eacb15f4c5482105ac0c05850b1 2024-11-11T22:18:10,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/e106382a03784649a8afce52d080b7ad is 50, key is test_row_0/C:col10/1731363489426/Put/seqid=0 2024-11-11T22:18:10,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742199_1375 (size=12151) 2024-11-11T22:18:10,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/e106382a03784649a8afce52d080b7ad 2024-11-11T22:18:10,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/67c3f235087e411cb3b5052a87fc23b7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/67c3f235087e411cb3b5052a87fc23b7 2024-11-11T22:18:10,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/67c3f235087e411cb3b5052a87fc23b7, entries=200, sequenceid=145, filesize=38.8 K 2024-11-11T22:18:10,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/a40b1eacb15f4c5482105ac0c05850b1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/a40b1eacb15f4c5482105ac0c05850b1 2024-11-11T22:18:10,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/a40b1eacb15f4c5482105ac0c05850b1, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T22:18:10,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/e106382a03784649a8afce52d080b7ad as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e106382a03784649a8afce52d080b7ad 2024-11-11T22:18:10,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e106382a03784649a8afce52d080b7ad, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T22:18:10,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76df2be919cd8ff2500e96d76dee493a in 1292ms, sequenceid=145, compaction requested=false 2024-11-11T22:18:10,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:10,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T22:18:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:10,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111126448a7546b84d36af0b10f975b07d1f_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363489525/Put/seqid=0 2024-11-11T22:18:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363550739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:18:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363550740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742200_1376 (size=17284) 2024-11-11T22:18:10,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363550748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363550749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363550859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363550860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363550869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:10,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:10,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:10,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363550879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:11,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363551080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:11,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363551080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363551089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363551099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,155 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:11,159 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111126448a7546b84d36af0b10f975b07d1f_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111126448a7546b84d36af0b10f975b07d1f_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:11,159 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/6a8ed7965ed34c1f9672df55e58f620b, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:11,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/6a8ed7965ed34c1f9672df55e58f620b is 175, key is test_row_0/A:col10/1731363489525/Put/seqid=0 2024-11-11T22:18:11,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742201_1377 (size=48389) 2024-11-11T22:18:11,168 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/6a8ed7965ed34c1f9672df55e58f620b 2024-11-11T22:18:11,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/194c26aa99c8473bac8fc659730353ec is 50, key is test_row_0/B:col10/1731363489525/Put/seqid=0 2024-11-11T22:18:11,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742202_1378 (size=12151) 2024-11-11T22:18:11,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:18:11,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:11,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:11,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363551390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363551399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363551409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363551419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:11,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:11,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:11,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:11,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:11,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/194c26aa99c8473bac8fc659730353ec 2024-11-11T22:18:11,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/80e0ff262f824669b65857e84cc2391f is 50, key is test_row_0/C:col10/1731363489525/Put/seqid=0 2024-11-11T22:18:11,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742203_1379 (size=12151) 2024-11-11T22:18:11,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/80e0ff262f824669b65857e84cc2391f 2024-11-11T22:18:11,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/6a8ed7965ed34c1f9672df55e58f620b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/6a8ed7965ed34c1f9672df55e58f620b 2024-11-11T22:18:11,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/6a8ed7965ed34c1f9672df55e58f620b, entries=250, sequenceid=184, filesize=47.3 K 2024-11-11T22:18:11,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/194c26aa99c8473bac8fc659730353ec as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/194c26aa99c8473bac8fc659730353ec 2024-11-11T22:18:11,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/194c26aa99c8473bac8fc659730353ec, entries=150, sequenceid=184, filesize=11.9 K 2024-11-11T22:18:11,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/80e0ff262f824669b65857e84cc2391f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/80e0ff262f824669b65857e84cc2391f 2024-11-11T22:18:11,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/80e0ff262f824669b65857e84cc2391f, entries=150, sequenceid=184, filesize=11.9 K 2024-11-11T22:18:11,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 76df2be919cd8ff2500e96d76dee493a in 874ms, sequenceid=184, compaction requested=true 2024-11-11T22:18:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:11,605 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:11,605 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:11,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:11,608 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:11,608 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/B is initiating minor compaction (all files) 2024-11-11T22:18:11,608 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:11,608 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/B in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,608 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/A is initiating minor compaction (all files) 2024-11-11T22:18:11,608 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/A in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,608 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3321b41534aa4fd1a128a76a69504554, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/a40b1eacb15f4c5482105ac0c05850b1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/194c26aa99c8473bac8fc659730353ec] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=35.8 K 2024-11-11T22:18:11,608 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ba8d322c097246c28cb63bcf80df993e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/67c3f235087e411cb3b5052a87fc23b7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/6a8ed7965ed34c1f9672df55e58f620b] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=116.6 K 2024-11-11T22:18:11,608 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,608 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ba8d322c097246c28cb63bcf80df993e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/67c3f235087e411cb3b5052a87fc23b7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/6a8ed7965ed34c1f9672df55e58f620b] 2024-11-11T22:18:11,609 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3321b41534aa4fd1a128a76a69504554, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363487860 2024-11-11T22:18:11,609 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba8d322c097246c28cb63bcf80df993e, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363487860 2024-11-11T22:18:11,609 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a40b1eacb15f4c5482105ac0c05850b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731363488228 2024-11-11T22:18:11,609 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67c3f235087e411cb3b5052a87fc23b7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731363488228 2024-11-11T22:18:11,609 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 194c26aa99c8473bac8fc659730353ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731363489525 2024-11-11T22:18:11,609 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8ed7965ed34c1f9672df55e58f620b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731363489525 2024-11-11T22:18:11,613 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:11,614 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:11,614 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112637e9c7011b43c58d6c531af99c8fcc_76df2be919cd8ff2500e96d76dee493a store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:11,615 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/bdb07940bd7a42808da0fff6fbe6beaa is 50, key is test_row_0/B:col10/1731363489525/Put/seqid=0 2024-11-11T22:18:11,616 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112637e9c7011b43c58d6c531af99c8fcc_76df2be919cd8ff2500e96d76dee493a, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:11,617 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112637e9c7011b43c58d6c531af99c8fcc_76df2be919cd8ff2500e96d76dee493a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:11,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742204_1380 (size=12493) 2024-11-11T22:18:11,624 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/bdb07940bd7a42808da0fff6fbe6beaa as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bdb07940bd7a42808da0fff6fbe6beaa 2024-11-11T22:18:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742205_1381 (size=4469) 2024-11-11T22:18:11,628 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/B of 76df2be919cd8ff2500e96d76dee493a into bdb07940bd7a42808da0fff6fbe6beaa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:11,628 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:11,628 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/B, priority=13, startTime=1731363491605; duration=0sec 2024-11-11T22:18:11,629 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:11,629 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:B 2024-11-11T22:18:11,629 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:11,629 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:11,630 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/C is initiating minor compaction (all files) 2024-11-11T22:18:11,630 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/C in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,630 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/12c45f8de1fe443c8dd7c36ac285584c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e106382a03784649a8afce52d080b7ad, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/80e0ff262f824669b65857e84cc2391f] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=35.8 K 2024-11-11T22:18:11,632 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 12c45f8de1fe443c8dd7c36ac285584c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363487860 2024-11-11T22:18:11,632 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e106382a03784649a8afce52d080b7ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731363488228 2024-11-11T22:18:11,633 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 80e0ff262f824669b65857e84cc2391f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731363489525 2024-11-11T22:18:11,638 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:11,638 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/38fdcb06b2ba49dd9a49e919a85914d8 is 50, key is test_row_0/C:col10/1731363489525/Put/seqid=0 2024-11-11T22:18:11,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T22:18:11,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:11,648 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T22:18:11,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742206_1382 (size=12493) 2024-11-11T22:18:11,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:11,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:11,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:11,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:11,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:11,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:11,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dc0970de3017496a93ec49ca44393d48_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363490746/Put/seqid=0 2024-11-11T22:18:11,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742207_1383 (size=12304) 2024-11-11T22:18:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:11,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363491946, max timestamp=1731363491964Num uniques 859; 2024-11-11T22:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363491946, max timestamp=1731363491964Num uniques 859; 2024-11-11T22:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363491946, max timestamp=1731363491964Num uniques 859; 2024-11-11T22:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:11,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:12,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363552005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363552006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363552010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363552011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,026 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:12,027 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/79a628a1dea34923ac3aef5471452442 is 175, key is test_row_0/A:col10/1731363489525/Put/seqid=0 2024-11-11T22:18:12,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742208_1384 (size=31447) 2024-11-11T22:18:12,052 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/38fdcb06b2ba49dd9a49e919a85914d8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/38fdcb06b2ba49dd9a49e919a85914d8 2024-11-11T22:18:12,056 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/C of 76df2be919cd8ff2500e96d76dee493a into 38fdcb06b2ba49dd9a49e919a85914d8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:12,056 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:12,056 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/C, priority=13, startTime=1731363491605; duration=0sec 2024-11-11T22:18:12,057 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:12,057 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:C 2024-11-11T22:18:12,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:12,073 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111dc0970de3017496a93ec49ca44393d48_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc0970de3017496a93ec49ca44393d48_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:12,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/27b41abc7b2d4b98a12c94d99fac09fc, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:12,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/27b41abc7b2d4b98a12c94d99fac09fc is 175, key is test_row_0/A:col10/1731363490746/Put/seqid=0 2024-11-11T22:18:12,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742209_1385 (size=31105) 2024-11-11T22:18:12,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363552129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363552129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363552130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363552130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:18:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363552339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363552340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363552340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363552340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,436 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/79a628a1dea34923ac3aef5471452442 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/79a628a1dea34923ac3aef5471452442 2024-11-11T22:18:12,440 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/A of 76df2be919cd8ff2500e96d76dee493a into 79a628a1dea34923ac3aef5471452442(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:12,440 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:12,440 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/A, priority=13, startTime=1731363491605; duration=0sec 2024-11-11T22:18:12,441 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:12,441 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:A 2024-11-11T22:18:12,482 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/27b41abc7b2d4b98a12c94d99fac09fc 2024-11-11T22:18:12,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/958748fb498a4ab99920f8f838f9d58a is 50, key is test_row_0/B:col10/1731363490746/Put/seqid=0 2024-11-11T22:18:12,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742210_1386 (size=12151) 2024-11-11T22:18:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363552660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363552660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363552660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363552660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:12,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/958748fb498a4ab99920f8f838f9d58a 2024-11-11T22:18:12,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/1d3fc00522d344dcbb954d9105d4cc1d is 50, key is test_row_0/C:col10/1731363490746/Put/seqid=0 2024-11-11T22:18:12,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742211_1387 (size=12151) 2024-11-11T22:18:12,901 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/1d3fc00522d344dcbb954d9105d4cc1d 2024-11-11T22:18:12,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/27b41abc7b2d4b98a12c94d99fac09fc as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27b41abc7b2d4b98a12c94d99fac09fc 2024-11-11T22:18:12,907 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27b41abc7b2d4b98a12c94d99fac09fc, entries=150, sequenceid=194, filesize=30.4 K 2024-11-11T22:18:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/958748fb498a4ab99920f8f838f9d58a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/958748fb498a4ab99920f8f838f9d58a 2024-11-11T22:18:12,912 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/958748fb498a4ab99920f8f838f9d58a, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T22:18:12,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/1d3fc00522d344dcbb954d9105d4cc1d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/1d3fc00522d344dcbb954d9105d4cc1d 2024-11-11T22:18:12,924 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/1d3fc00522d344dcbb954d9105d4cc1d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T22:18:12,925 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 76df2be919cd8ff2500e96d76dee493a in 1277ms, sequenceid=194, compaction requested=false 2024-11-11T22:18:12,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:12,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:12,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T22:18:12,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T22:18:12,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T22:18:12,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8050 sec 2024-11-11T22:18:12,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.8120 sec 2024-11-11T22:18:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:13,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T22:18:13,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:13,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T22:18:13,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:13,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T22:18:13,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:13,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T22:18:13,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111973c6f9c9a5942b0baef720caefade3e_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363492005/Put/seqid=0 2024-11-11T22:18:13,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742212_1388 (size=17284) 2024-11-11T22:18:13,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363553189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363553191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363553194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363553207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363553319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363553319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363553319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363553319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363553530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363553530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363553530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363553530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,605 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:13,608 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111973c6f9c9a5942b0baef720caefade3e_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111973c6f9c9a5942b0baef720caefade3e_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:13,608 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ab378b70e5e146c49788ca956655f67e, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:13,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ab378b70e5e146c49788ca956655f67e is 175, key is test_row_0/A:col10/1731363492005/Put/seqid=0 2024-11-11T22:18:13,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742213_1389 (size=48389) 2024-11-11T22:18:13,613 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ab378b70e5e146c49788ca956655f67e 2024-11-11T22:18:13,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/09c9ee7cfc644323b9f7a72d4b360ea4 is 50, key is test_row_0/B:col10/1731363492005/Put/seqid=0 2024-11-11T22:18:13,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742214_1390 (size=12151) 2024-11-11T22:18:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363553849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363553850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363553850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363553850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:14,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/09c9ee7cfc644323b9f7a72d4b360ea4 2024-11-11T22:18:14,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/4d5993e854854bd3ae48f8c8efe4e1f8 is 50, key is test_row_0/C:col10/1731363492005/Put/seqid=0 2024-11-11T22:18:14,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742215_1391 (size=12151) 2024-11-11T22:18:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T22:18:14,290 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:14,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T22:18:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:18:14,292 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:14,292 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:14,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:14,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:14,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363554369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363554369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:14,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:14,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363554370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363554370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:18:14,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/4d5993e854854bd3ae48f8c8efe4e1f8 2024-11-11T22:18:14,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ab378b70e5e146c49788ca956655f67e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab378b70e5e146c49788ca956655f67e 2024-11-11T22:18:14,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab378b70e5e146c49788ca956655f67e, entries=250, sequenceid=235, filesize=47.3 K 2024-11-11T22:18:14,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/09c9ee7cfc644323b9f7a72d4b360ea4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/09c9ee7cfc644323b9f7a72d4b360ea4 2024-11-11T22:18:14,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:18:14,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:14,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:14,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:14,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:14,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/09c9ee7cfc644323b9f7a72d4b360ea4, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T22:18:14,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/4d5993e854854bd3ae48f8c8efe4e1f8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4d5993e854854bd3ae48f8c8efe4e1f8 2024-11-11T22:18:14,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4d5993e854854bd3ae48f8c8efe4e1f8, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T22:18:14,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for 76df2be919cd8ff2500e96d76dee493a in 1265ms, sequenceid=235, compaction requested=true 2024-11-11T22:18:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:14,451 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:14,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:14,451 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:14,452 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:14,452 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:14,452 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/A is initiating minor compaction (all files) 2024-11-11T22:18:14,452 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/B is initiating minor compaction (all files) 2024-11-11T22:18:14,452 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/A in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:14,452 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/B in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:14,452 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/79a628a1dea34923ac3aef5471452442, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27b41abc7b2d4b98a12c94d99fac09fc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab378b70e5e146c49788ca956655f67e] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=108.3 K 2024-11-11T22:18:14,452 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bdb07940bd7a42808da0fff6fbe6beaa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/958748fb498a4ab99920f8f838f9d58a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/09c9ee7cfc644323b9f7a72d4b360ea4] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=35.9 K 2024-11-11T22:18:14,452 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:14,452 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/79a628a1dea34923ac3aef5471452442, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27b41abc7b2d4b98a12c94d99fac09fc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab378b70e5e146c49788ca956655f67e] 2024-11-11T22:18:14,453 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb07940bd7a42808da0fff6fbe6beaa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731363489525 2024-11-11T22:18:14,453 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79a628a1dea34923ac3aef5471452442, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731363489525 2024-11-11T22:18:14,453 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 958748fb498a4ab99920f8f838f9d58a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731363490739 2024-11-11T22:18:14,453 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b41abc7b2d4b98a12c94d99fac09fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731363490739 2024-11-11T22:18:14,453 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c9ee7cfc644323b9f7a72d4b360ea4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731363492005 2024-11-11T22:18:14,453 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab378b70e5e146c49788ca956655f67e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731363492004 2024-11-11T22:18:14,462 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:14,464 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:14,464 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/6b6e32a6c2ce4c5fbb1ff524db56d66f is 50, key is test_row_0/B:col10/1731363492005/Put/seqid=0 2024-11-11T22:18:14,469 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111152b55da53dcf4d5eb42dfebaddef9752_76df2be919cd8ff2500e96d76dee493a store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:14,471 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111152b55da53dcf4d5eb42dfebaddef9752_76df2be919cd8ff2500e96d76dee493a, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:14,471 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111152b55da53dcf4d5eb42dfebaddef9752_76df2be919cd8ff2500e96d76dee493a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:14,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742216_1392 (size=12595) 2024-11-11T22:18:14,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742217_1393 (size=4469) 2024-11-11T22:18:14,500 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#A#compaction#334 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:14,500 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/4734a9cdb5dd417cb142841af5f38e1d is 175, key is test_row_0/A:col10/1731363492005/Put/seqid=0 2024-11-11T22:18:14,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742218_1394 (size=31549) 2024-11-11T22:18:14,554 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/4734a9cdb5dd417cb142841af5f38e1d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4734a9cdb5dd417cb142841af5f38e1d 2024-11-11T22:18:14,559 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/A of 76df2be919cd8ff2500e96d76dee493a into 4734a9cdb5dd417cb142841af5f38e1d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:14,559 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:14,560 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/A, priority=13, startTime=1731363494451; duration=0sec 2024-11-11T22:18:14,560 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:14,560 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:A 2024-11-11T22:18:14,560 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:14,561 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:14,561 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/C is initiating minor compaction (all files) 2024-11-11T22:18:14,561 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/C in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:14,561 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/38fdcb06b2ba49dd9a49e919a85914d8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/1d3fc00522d344dcbb954d9105d4cc1d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4d5993e854854bd3ae48f8c8efe4e1f8] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=35.9 K 2024-11-11T22:18:14,561 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38fdcb06b2ba49dd9a49e919a85914d8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731363489525 2024-11-11T22:18:14,562 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d3fc00522d344dcbb954d9105d4cc1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731363490739 2024-11-11T22:18:14,563 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d5993e854854bd3ae48f8c8efe4e1f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731363492005 2024-11-11T22:18:14,570 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:14,570 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/e4c74e1a97de43fbae78c2b248127274 is 50, key is test_row_0/C:col10/1731363492005/Put/seqid=0 2024-11-11T22:18:14,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742219_1395 (size=12595) 2024-11-11T22:18:14,589 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/e4c74e1a97de43fbae78c2b248127274 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e4c74e1a97de43fbae78c2b248127274 2024-11-11T22:18:14,595 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/C of 76df2be919cd8ff2500e96d76dee493a into e4c74e1a97de43fbae78c2b248127274(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:14,595 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:14,595 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/C, priority=13, startTime=1731363494451; duration=0sec 2024-11-11T22:18:14,595 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:14,595 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:C 2024-11-11T22:18:14,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T22:18:14,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:14,597 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T22:18:14,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:14,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:14,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:14,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:14,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:14,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:14,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111106f65cfc6d204c0bba292dd3959c861a_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363493206/Put/seqid=0 2024-11-11T22:18:14,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742220_1396 (size=12304) 2024-11-11T22:18:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:18:14,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:14,615 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111106f65cfc6d204c0bba292dd3959c861a_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106f65cfc6d204c0bba292dd3959c861a_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:14,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/9e226c052ce14a30aff2742effaedd9d, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:14,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/9e226c052ce14a30aff2742effaedd9d is 175, key is test_row_0/A:col10/1731363493206/Put/seqid=0 2024-11-11T22:18:14,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742221_1397 (size=31105) 2024-11-11T22:18:14,876 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/6b6e32a6c2ce4c5fbb1ff524db56d66f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/6b6e32a6c2ce4c5fbb1ff524db56d66f 2024-11-11T22:18:14,879 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/B of 76df2be919cd8ff2500e96d76dee493a into 6b6e32a6c2ce4c5fbb1ff524db56d66f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:14,879 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:14,879 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/B, priority=13, startTime=1731363494451; duration=0sec 2024-11-11T22:18:14,880 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:14,880 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:B 2024-11-11T22:18:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:18:15,020 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/9e226c052ce14a30aff2742effaedd9d 2024-11-11T22:18:15,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/43c60a6987a04fa7ad78a2094c16ce0f is 50, key is test_row_0/B:col10/1731363493206/Put/seqid=0 2024-11-11T22:18:15,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742222_1398 (size=12151) 2024-11-11T22:18:15,067 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/43c60a6987a04fa7ad78a2094c16ce0f 2024-11-11T22:18:15,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/2f6e849e6e4e4dabb238692790b0ce0c is 50, key is test_row_0/C:col10/1731363493206/Put/seqid=0 2024-11-11T22:18:15,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742223_1399 (size=12151) 2024-11-11T22:18:15,083 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/2f6e849e6e4e4dabb238692790b0ce0c 2024-11-11T22:18:15,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/9e226c052ce14a30aff2742effaedd9d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/9e226c052ce14a30aff2742effaedd9d 2024-11-11T22:18:15,094 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/9e226c052ce14a30aff2742effaedd9d, entries=150, sequenceid=244, filesize=30.4 K 2024-11-11T22:18:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/43c60a6987a04fa7ad78a2094c16ce0f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/43c60a6987a04fa7ad78a2094c16ce0f 2024-11-11T22:18:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,099 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/43c60a6987a04fa7ad78a2094c16ce0f, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T22:18:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/2f6e849e6e4e4dabb238692790b0ce0c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2f6e849e6e4e4dabb238692790b0ce0c 2024-11-11T22:18:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,103 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2f6e849e6e4e4dabb238692790b0ce0c, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T22:18:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 76df2be919cd8ff2500e96d76dee493a in 507ms, sequenceid=244, compaction requested=false 2024-11-11T22:18:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T22:18:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T22:18:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T22:18:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 813 msec 2024-11-11T22:18:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 818 msec 2024-11-11T22:18:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T22:18:15,429 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-11T22:18:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:18:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,433 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:15,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:15,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:18:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:15,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:15,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b6ba78c0858a4f909e8b7d763861bd56_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363495484/Put/seqid=0 2024-11-11T22:18:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742224_1400 (size=22268) 2024-11-11T22:18:15,522 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:15,525 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b6ba78c0858a4f909e8b7d763861bd56_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b6ba78c0858a4f909e8b7d763861bd56_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:15,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1d37221ac9bf48108a0900496f534dcf, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:15,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1d37221ac9bf48108a0900496f534dcf is 175, key is test_row_0/A:col10/1731363495484/Put/seqid=0 2024-11-11T22:18:15,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742225_1401 (size=65673) 2024-11-11T22:18:15,534 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1d37221ac9bf48108a0900496f534dcf 2024-11-11T22:18:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:18:15,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/bf6b3f2826a64b0989d4ee43b52b2e45 is 50, key is test_row_0/B:col10/1731363495484/Put/seqid=0 2024-11-11T22:18:15,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742226_1402 (size=12151) 2024-11-11T22:18:15,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731363495516, max timestamp=1731363495553Num uniques 859; 2024-11-11T22:18:15,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731363495516, max timestamp=1731363495553Num uniques 859; 2024-11-11T22:18:15,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T22:18:15,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731363495516, max timestamp=1731363495553Num uniques 859; 2024-11-11T22:18:15,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T22:18:15,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T22:18:15,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T22:18:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:15,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363555612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363555618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363555619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363555621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T22:18:15,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:15,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:15,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:15,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:18:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363555739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363555749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363555749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363555749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T22:18:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:15,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:15,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/bf6b3f2826a64b0989d4ee43b52b2e45 2024-11-11T22:18:15,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/06411c9c2dfc434bbb52a32088b7fef3 is 50, key is test_row_0/C:col10/1731363495484/Put/seqid=0 2024-11-11T22:18:15,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742227_1403 (size=12151) 2024-11-11T22:18:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363555959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363555960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363555960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363555960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:15,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T22:18:16,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:16,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:18:16,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T22:18:16,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:16,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363556280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363556280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363556280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363556280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/06411c9c2dfc434bbb52a32088b7fef3 2024-11-11T22:18:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1d37221ac9bf48108a0900496f534dcf as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1d37221ac9bf48108a0900496f534dcf 2024-11-11T22:18:16,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1d37221ac9bf48108a0900496f534dcf, entries=350, sequenceid=255, filesize=64.1 K 2024-11-11T22:18:16,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T22:18:16,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/bf6b3f2826a64b0989d4ee43b52b2e45 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bf6b3f2826a64b0989d4ee43b52b2e45 2024-11-11T22:18:16,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:16,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:16,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bf6b3f2826a64b0989d4ee43b52b2e45, entries=150, sequenceid=255, filesize=11.9 K 2024-11-11T22:18:16,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/06411c9c2dfc434bbb52a32088b7fef3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/06411c9c2dfc434bbb52a32088b7fef3 2024-11-11T22:18:16,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/06411c9c2dfc434bbb52a32088b7fef3, entries=150, sequenceid=255, filesize=11.9 K 2024-11-11T22:18:16,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 76df2be919cd8ff2500e96d76dee493a in 882ms, sequenceid=255, compaction requested=true 2024-11-11T22:18:16,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:16,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:16,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:16,369 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:16,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:16,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:16,369 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:16,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:16,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128327 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/B is initiating minor compaction (all files) 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/A is initiating minor compaction (all files) 2024-11-11T22:18:16,370 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/A in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,370 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/B in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,370 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/6b6e32a6c2ce4c5fbb1ff524db56d66f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/43c60a6987a04fa7ad78a2094c16ce0f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bf6b3f2826a64b0989d4ee43b52b2e45] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=36.0 K 2024-11-11T22:18:16,370 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4734a9cdb5dd417cb142841af5f38e1d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/9e226c052ce14a30aff2742effaedd9d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1d37221ac9bf48108a0900496f534dcf] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=125.3 K 2024-11-11T22:18:16,370 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4734a9cdb5dd417cb142841af5f38e1d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/9e226c052ce14a30aff2742effaedd9d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1d37221ac9bf48108a0900496f534dcf] 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b6e32a6c2ce4c5fbb1ff524db56d66f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731363492005 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4734a9cdb5dd417cb142841af5f38e1d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731363492005 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c60a6987a04fa7ad78a2094c16ce0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731363493189 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e226c052ce14a30aff2742effaedd9d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731363493189 2024-11-11T22:18:16,370 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bf6b3f2826a64b0989d4ee43b52b2e45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731363495479 2024-11-11T22:18:16,371 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d37221ac9bf48108a0900496f534dcf, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731363495440 2024-11-11T22:18:16,378 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:16,380 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111bc9440275aca4cb8807afbc05eda84c8_76df2be919cd8ff2500e96d76dee493a store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:16,380 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#B#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:16,381 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/ccebb84b2a05438cbe44468d997db72d is 50, key is test_row_0/B:col10/1731363495484/Put/seqid=0 2024-11-11T22:18:16,382 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111bc9440275aca4cb8807afbc05eda84c8_76df2be919cd8ff2500e96d76dee493a, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:16,382 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bc9440275aca4cb8807afbc05eda84c8_76df2be919cd8ff2500e96d76dee493a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:16,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742228_1404 (size=12697) 2024-11-11T22:18:16,393 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/ccebb84b2a05438cbe44468d997db72d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ccebb84b2a05438cbe44468d997db72d 2024-11-11T22:18:16,398 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/B of 76df2be919cd8ff2500e96d76dee493a into ccebb84b2a05438cbe44468d997db72d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:16,398 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:16,398 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/B, priority=13, startTime=1731363496369; duration=0sec 2024-11-11T22:18:16,398 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:16,399 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:B 2024-11-11T22:18:16,399 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:16,399 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:16,399 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/C is initiating minor compaction (all files) 2024-11-11T22:18:16,400 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/C in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,400 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e4c74e1a97de43fbae78c2b248127274, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2f6e849e6e4e4dabb238692790b0ce0c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/06411c9c2dfc434bbb52a32088b7fef3] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=36.0 K 2024-11-11T22:18:16,400 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c74e1a97de43fbae78c2b248127274, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731363492005 2024-11-11T22:18:16,401 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f6e849e6e4e4dabb238692790b0ce0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731363493189 2024-11-11T22:18:16,401 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 06411c9c2dfc434bbb52a32088b7fef3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731363495479 2024-11-11T22:18:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742229_1405 (size=4469) 2024-11-11T22:18:16,406 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#C#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:16,407 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/6a68ccb5f05c4852b2dfb3b346dfdf98 is 50, key is test_row_0/C:col10/1731363495484/Put/seqid=0 2024-11-11T22:18:16,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742230_1406 (size=12697) 2024-11-11T22:18:16,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T22:18:16,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:16,514 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-11T22:18:16,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:16,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:16,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:16,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:16,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:16,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:16,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112fec6aeb70604dc980bba5de9fbc1a6b_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363495618/Put/seqid=0 2024-11-11T22:18:16,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742231_1407 (size=12454) 2024-11-11T22:18:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:18:16,803 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#A#compaction#342 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:16,804 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/3b34721929934573b87eb488ecae2c78 is 175, key is test_row_0/A:col10/1731363495484/Put/seqid=0 2024-11-11T22:18:16,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742232_1408 (size=31651) 2024-11-11T22:18:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:16,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:16,815 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/6a68ccb5f05c4852b2dfb3b346dfdf98 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/6a68ccb5f05c4852b2dfb3b346dfdf98 2024-11-11T22:18:16,818 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/C of 76df2be919cd8ff2500e96d76dee493a into 6a68ccb5f05c4852b2dfb3b346dfdf98(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:16,818 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:16,818 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/C, priority=13, startTime=1731363496369; duration=0sec 2024-11-11T22:18:16,819 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:16,819 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:C 2024-11-11T22:18:16,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363556813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363556819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363556820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363556826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:16,937 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112fec6aeb70604dc980bba5de9fbc1a6b_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112fec6aeb70604dc980bba5de9fbc1a6b_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ab9ca37023014f409eee9373964ca48e, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:16,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ab9ca37023014f409eee9373964ca48e is 175, key is test_row_0/A:col10/1731363495618/Put/seqid=0 2024-11-11T22:18:16,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742233_1409 (size=31255) 2024-11-11T22:18:16,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363556939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363556939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363556939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363556940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:16,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:16,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363557159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363557159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363557160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363557160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,211 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/3b34721929934573b87eb488ecae2c78 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3b34721929934573b87eb488ecae2c78 2024-11-11T22:18:17,215 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/A of 76df2be919cd8ff2500e96d76dee493a into 3b34721929934573b87eb488ecae2c78(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:17,216 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:17,216 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/A, priority=13, startTime=1731363496369; duration=0sec 2024-11-11T22:18:17,216 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:17,216 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:A 2024-11-11T22:18:17,342 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ab9ca37023014f409eee9373964ca48e 2024-11-11T22:18:17,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/655ebbd99f9f4c438aa59a4d0bf92fa0 is 50, key is test_row_0/B:col10/1731363495618/Put/seqid=0 2024-11-11T22:18:17,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742234_1410 (size=12301) 2024-11-11T22:18:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363557469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363557469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363557469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363557469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:18:17,752 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/655ebbd99f9f4c438aa59a4d0bf92fa0 2024-11-11T22:18:17,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/383b48dd51284a169b8bda81679f717e is 50, key is test_row_0/C:col10/1731363495618/Put/seqid=0 2024-11-11T22:18:17,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742235_1411 (size=12301) 2024-11-11T22:18:17,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363557979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363557980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363557989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363557989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:18,161 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/383b48dd51284a169b8bda81679f717e 2024-11-11T22:18:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/ab9ca37023014f409eee9373964ca48e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab9ca37023014f409eee9373964ca48e 2024-11-11T22:18:18,168 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab9ca37023014f409eee9373964ca48e, entries=150, sequenceid=291, filesize=30.5 K 2024-11-11T22:18:18,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/655ebbd99f9f4c438aa59a4d0bf92fa0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/655ebbd99f9f4c438aa59a4d0bf92fa0 2024-11-11T22:18:18,171 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/655ebbd99f9f4c438aa59a4d0bf92fa0, entries=150, sequenceid=291, filesize=12.0 K 2024-11-11T22:18:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/383b48dd51284a169b8bda81679f717e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/383b48dd51284a169b8bda81679f717e 2024-11-11T22:18:18,175 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/383b48dd51284a169b8bda81679f717e, entries=150, sequenceid=291, filesize=12.0 K 2024-11-11T22:18:18,176 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 76df2be919cd8ff2500e96d76dee493a in 1663ms, sequenceid=291, compaction requested=false 2024-11-11T22:18:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:18,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-11T22:18:18,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-11T22:18:18,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T22:18:18,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7470 sec 2024-11-11T22:18:18,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7520 sec 2024-11-11T22:18:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:18,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T22:18:18,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:18,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:18,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:18,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:18,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:18,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:18,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a78476b05399484592d49b048d85f66b_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363496817/Put/seqid=0 2024-11-11T22:18:19,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742236_1412 (size=14994) 2024-11-11T22:18:19,004 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:19,007 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a78476b05399484592d49b048d85f66b_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a78476b05399484592d49b048d85f66b_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:19,008 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/32f5aeea2d18428ba77b36b9e4d94d35, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:19,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/32f5aeea2d18428ba77b36b9e4d94d35 is 175, key is test_row_0/A:col10/1731363496817/Put/seqid=0 2024-11-11T22:18:19,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742237_1413 (size=39949) 2024-11-11T22:18:19,012 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/32f5aeea2d18428ba77b36b9e4d94d35 2024-11-11T22:18:19,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/9f641bea48d84a61a6c8fe8162753c6c is 50, key is test_row_0/B:col10/1731363496817/Put/seqid=0 2024-11-11T22:18:19,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742238_1414 (size=12301) 2024-11-11T22:18:19,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731363499013, max timestamp=1731363499044Num uniques 859; 2024-11-11T22:18:19,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T22:18:19,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731363499013, max timestamp=1731363499044Num uniques 859; 2024-11-11T22:18:19,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731363499013, max timestamp=1731363499044Num uniques 859; 2024-11-11T22:18:19,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T22:18:19,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T22:18:19,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363559081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363559082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363559093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363559098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363559209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363559209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363559209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363559219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363559419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363559419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363559419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/9f641bea48d84a61a6c8fe8162753c6c 2024-11-11T22:18:19,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/fcb043fc521841329ae6eb35c40be398 is 50, key is test_row_0/C:col10/1731363496817/Put/seqid=0 2024-11-11T22:18:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363559439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742239_1415 (size=12301) 2024-11-11T22:18:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T22:18:19,589 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:19,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-11T22:18:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T22:18:19,591 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:19,592 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:19,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:19,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T22:18:19,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T22:18:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:19,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363559739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363559739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363559739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363559750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:19,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/fcb043fc521841329ae6eb35c40be398 2024-11-11T22:18:19,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/32f5aeea2d18428ba77b36b9e4d94d35 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/32f5aeea2d18428ba77b36b9e4d94d35 2024-11-11T22:18:19,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/32f5aeea2d18428ba77b36b9e4d94d35, entries=200, sequenceid=304, filesize=39.0 K 2024-11-11T22:18:19,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/9f641bea48d84a61a6c8fe8162753c6c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9f641bea48d84a61a6c8fe8162753c6c 2024-11-11T22:18:19,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9f641bea48d84a61a6c8fe8162753c6c, entries=150, sequenceid=304, filesize=12.0 K 2024-11-11T22:18:19,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/fcb043fc521841329ae6eb35c40be398 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fcb043fc521841329ae6eb35c40be398 2024-11-11T22:18:19,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fcb043fc521841329ae6eb35c40be398, entries=150, sequenceid=304, filesize=12.0 K 2024-11-11T22:18:19,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 76df2be919cd8ff2500e96d76dee493a in 870ms, sequenceid=304, compaction requested=true 2024-11-11T22:18:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:19,862 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:19,862 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:19,863 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:19,863 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:19,863 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/A is initiating minor compaction (all files) 2024-11-11T22:18:19,863 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/B is initiating minor compaction (all files) 2024-11-11T22:18:19,863 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/A in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:19,863 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/B in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:19,863 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3b34721929934573b87eb488ecae2c78, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab9ca37023014f409eee9373964ca48e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/32f5aeea2d18428ba77b36b9e4d94d35] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=100.4 K 2024-11-11T22:18:19,863 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ccebb84b2a05438cbe44468d997db72d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/655ebbd99f9f4c438aa59a4d0bf92fa0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9f641bea48d84a61a6c8fe8162753c6c] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=36.4 K 2024-11-11T22:18:19,863 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:19,863 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3b34721929934573b87eb488ecae2c78, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab9ca37023014f409eee9373964ca48e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/32f5aeea2d18428ba77b36b9e4d94d35] 2024-11-11T22:18:19,863 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ccebb84b2a05438cbe44468d997db72d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731363495479 2024-11-11T22:18:19,863 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b34721929934573b87eb488ecae2c78, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731363495479 2024-11-11T22:18:19,864 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 655ebbd99f9f4c438aa59a4d0bf92fa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731363495580 2024-11-11T22:18:19,864 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab9ca37023014f409eee9373964ca48e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731363495580 2024-11-11T22:18:19,864 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f641bea48d84a61a6c8fe8162753c6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363496817 2024-11-11T22:18:19,864 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32f5aeea2d18428ba77b36b9e4d94d35, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363496817 2024-11-11T22:18:19,870 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:19,871 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/3096c78cef26408b8bc0c07b41be9e94 is 50, key is test_row_0/B:col10/1731363496817/Put/seqid=0 2024-11-11T22:18:19,873 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:19,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742240_1416 (size=12949) 2024-11-11T22:18:19,877 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c1c8f49a4ed84659b23f98623a0c4a37_76df2be919cd8ff2500e96d76dee493a store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:19,879 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c1c8f49a4ed84659b23f98623a0c4a37_76df2be919cd8ff2500e96d76dee493a, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:19,879 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c1c8f49a4ed84659b23f98623a0c4a37_76df2be919cd8ff2500e96d76dee493a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:19,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742241_1417 (size=4469) 2024-11-11T22:18:19,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T22:18:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:19,897 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-11T22:18:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:19,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:19,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111cb987e985f74a8b9e7bec07978346bb_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363499081/Put/seqid=0 2024-11-11T22:18:19,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742242_1418 (size=12454) 2024-11-11T22:18:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T22:18:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:19,915 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111cb987e985f74a8b9e7bec07978346bb_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111cb987e985f74a8b9e7bec07978346bb_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:19,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/d8996bbbbe6c48459586925a9e954e72, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:19,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/d8996bbbbe6c48459586925a9e954e72 is 175, key is test_row_0/A:col10/1731363499081/Put/seqid=0 2024-11-11T22:18:19,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742243_1419 (size=31255) 2024-11-11T22:18:19,919 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/d8996bbbbe6c48459586925a9e954e72 2024-11-11T22:18:19,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/9d171b98a3c2456487ac426d0d433133 is 50, key is test_row_0/B:col10/1731363499081/Put/seqid=0 2024-11-11T22:18:19,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742244_1420 (size=12301) 2024-11-11T22:18:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T22:18:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:20,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:20,278 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/3096c78cef26408b8bc0c07b41be9e94 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3096c78cef26408b8bc0c07b41be9e94 2024-11-11T22:18:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363560267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363560269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363560271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363560271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,282 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/B of 76df2be919cd8ff2500e96d76dee493a into 3096c78cef26408b8bc0c07b41be9e94(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:20,282 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:20,282 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/B, priority=13, startTime=1731363499862; duration=0sec 2024-11-11T22:18:20,282 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:20,282 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:B 2024-11-11T22:18:20,282 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:20,283 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:20,283 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:20,284 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/C is initiating minor compaction (all files) 2024-11-11T22:18:20,284 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/C in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:20,284 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/6a68ccb5f05c4852b2dfb3b346dfdf98, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/383b48dd51284a169b8bda81679f717e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fcb043fc521841329ae6eb35c40be398] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=36.4 K 2024-11-11T22:18:20,284 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/a8e5d6d323db45c49510e6931e0097a3 is 175, key is test_row_0/A:col10/1731363496817/Put/seqid=0 2024-11-11T22:18:20,284 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a68ccb5f05c4852b2dfb3b346dfdf98, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731363495479 2024-11-11T22:18:20,284 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 383b48dd51284a169b8bda81679f717e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731363495580 2024-11-11T22:18:20,284 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fcb043fc521841329ae6eb35c40be398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363496817 2024-11-11T22:18:20,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363560276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742245_1421 (size=31903) 2024-11-11T22:18:20,299 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#C#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:20,299 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/2e40fea53a514f20b820908d7bb06ceb is 50, key is test_row_0/C:col10/1731363496817/Put/seqid=0 2024-11-11T22:18:20,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742246_1422 (size=12949) 2024-11-11T22:18:20,330 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/9d171b98a3c2456487ac426d0d433133 2024-11-11T22:18:20,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/4628e3fa4881429d983db2aba75ef099 is 50, key is test_row_0/C:col10/1731363499081/Put/seqid=0 2024-11-11T22:18:20,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742247_1423 (size=12301) 2024-11-11T22:18:20,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363560389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363560390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363560390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363560399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363560601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363560601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363560601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363560620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,702 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/a8e5d6d323db45c49510e6931e0097a3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/a8e5d6d323db45c49510e6931e0097a3 2024-11-11T22:18:20,712 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/2e40fea53a514f20b820908d7bb06ceb as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2e40fea53a514f20b820908d7bb06ceb 2024-11-11T22:18:20,713 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/A of 76df2be919cd8ff2500e96d76dee493a into a8e5d6d323db45c49510e6931e0097a3(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:20,713 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:20,713 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/A, priority=13, startTime=1731363499862; duration=0sec 2024-11-11T22:18:20,713 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:20,713 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:A 2024-11-11T22:18:20,716 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/C of 76df2be919cd8ff2500e96d76dee493a into 2e40fea53a514f20b820908d7bb06ceb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:20,716 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:20,716 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/C, priority=13, startTime=1731363499862; duration=0sec 2024-11-11T22:18:20,716 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:20,716 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:C 2024-11-11T22:18:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T22:18:20,740 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/4628e3fa4881429d983db2aba75ef099 2024-11-11T22:18:20,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/d8996bbbbe6c48459586925a9e954e72 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/d8996bbbbe6c48459586925a9e954e72 2024-11-11T22:18:20,746 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/d8996bbbbe6c48459586925a9e954e72, entries=150, sequenceid=338, filesize=30.5 K 2024-11-11T22:18:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/9d171b98a3c2456487ac426d0d433133 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9d171b98a3c2456487ac426d0d433133 2024-11-11T22:18:20,750 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9d171b98a3c2456487ac426d0d433133, entries=150, sequenceid=338, filesize=12.0 K 2024-11-11T22:18:20,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/4628e3fa4881429d983db2aba75ef099 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4628e3fa4881429d983db2aba75ef099 2024-11-11T22:18:20,756 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4628e3fa4881429d983db2aba75ef099, entries=150, sequenceid=338, filesize=12.0 K 2024-11-11T22:18:20,757 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 76df2be919cd8ff2500e96d76dee493a in 860ms, sequenceid=338, compaction requested=false 2024-11-11T22:18:20,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:20,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:20,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-11T22:18:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-11T22:18:20,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T22:18:20,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1660 sec 2024-11-11T22:18:20,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.1720 sec 2024-11-11T22:18:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:20,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:18:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:20,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110a154c05db85437aaef1823ca9fb584d_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363500913/Put/seqid=0 2024-11-11T22:18:20,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742248_1424 (size=14994) 2024-11-11T22:18:20,926 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:20,929 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110a154c05db85437aaef1823ca9fb584d_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a154c05db85437aaef1823ca9fb584d_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:20,929 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/f22bc8467e2b477eb07b0959e81c0763, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:20,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/f22bc8467e2b477eb07b0959e81c0763 is 175, key is test_row_0/A:col10/1731363500913/Put/seqid=0 2024-11-11T22:18:20,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742249_1425 (size=39949) 2024-11-11T22:18:20,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363560950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363560956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363560957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363560959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:20,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363561069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363561069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363561079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363561079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363561280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363561280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33684 deadline: 1731363561289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363561289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,300 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_1' in 76df2be919cd8ff2500e96d76dee493a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:18:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363561299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,334 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/f22bc8467e2b477eb07b0959e81c0763 2024-11-11T22:18:21,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/7fe337655d804f128a11957842fc4510 is 50, key is test_row_0/B:col10/1731363500913/Put/seqid=0 2024-11-11T22:18:21,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742250_1426 (size=12301) 2024-11-11T22:18:21,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/7fe337655d804f128a11957842fc4510 2024-11-11T22:18:21,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/875ffca3ede04a168908d1359585d4e2 is 50, key is test_row_0/C:col10/1731363500913/Put/seqid=0 2024-11-11T22:18:21,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742251_1427 (size=12301) 2024-11-11T22:18:21,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33734 deadline: 1731363561600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33700 deadline: 1731363561600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33664 deadline: 1731363561609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33716 deadline: 1731363561620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:21,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T22:18:21,740 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:21,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T22:18:21,742 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T22:18:21,743 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:21,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:21,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/875ffca3ede04a168908d1359585d4e2 2024-11-11T22:18:21,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/f22bc8467e2b477eb07b0959e81c0763 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/f22bc8467e2b477eb07b0959e81c0763 2024-11-11T22:18:21,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/f22bc8467e2b477eb07b0959e81c0763, entries=200, sequenceid=353, filesize=39.0 K 2024-11-11T22:18:21,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/7fe337655d804f128a11957842fc4510 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/7fe337655d804f128a11957842fc4510 2024-11-11T22:18:21,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/7fe337655d804f128a11957842fc4510, entries=150, sequenceid=353, filesize=12.0 K 2024-11-11T22:18:21,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/875ffca3ede04a168908d1359585d4e2 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/875ffca3ede04a168908d1359585d4e2 2024-11-11T22:18:21,805 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-11T22:18:21,805 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:21,806 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,806 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:21,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/875ffca3ede04a168908d1359585d4e2, entries=150, sequenceid=353, filesize=12.0 K 2024-11-11T22:18:21,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 76df2be919cd8ff2500e96d76dee493a in 894ms, sequenceid=353, compaction requested=true 2024-11-11T22:18:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:21,808 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-11T22:18:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:21,808 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:21,808 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76df2be919cd8ff2500e96d76dee493a:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:21,808 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,808 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:21,808 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:21,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:21,809 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:21,809 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/B is initiating minor compaction (all files) 2024-11-11T22:18:21,809 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/B in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:21,809 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3096c78cef26408b8bc0c07b41be9e94, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9d171b98a3c2456487ac426d0d433133, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/7fe337655d804f128a11957842fc4510] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=36.7 K 2024-11-11T22:18:21,810 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:21,810 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/A is initiating minor compaction (all files) 2024-11-11T22:18:21,810 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/A in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:21,810 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/a8e5d6d323db45c49510e6931e0097a3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/d8996bbbbe6c48459586925a9e954e72, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/f22bc8467e2b477eb07b0959e81c0763] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=100.7 K 2024-11-11T22:18:21,810 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:21,810 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/a8e5d6d323db45c49510e6931e0097a3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/d8996bbbbe6c48459586925a9e954e72, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/f22bc8467e2b477eb07b0959e81c0763] 2024-11-11T22:18:21,810 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 3096c78cef26408b8bc0c07b41be9e94, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363496817 2024-11-11T22:18:21,811 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e5d6d323db45c49510e6931e0097a3, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363496817 2024-11-11T22:18:21,811 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d171b98a3c2456487ac426d0d433133, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731363499078 2024-11-11T22:18:21,811 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8996bbbbe6c48459586925a9e954e72, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731363499078 2024-11-11T22:18:21,811 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe337655d804f128a11957842fc4510, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731363500270 2024-11-11T22:18:21,811 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f22bc8467e2b477eb07b0959e81c0763, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731363500269 2024-11-11T22:18:21,813 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-11T22:18:21,813 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:21,813 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,814 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:21,816 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-11T22:18:21,816 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:21,816 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:21,816 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,818 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:21,819 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-11T22:18:21,819 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:21,819 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,819 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:21,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:21,819 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:21,820 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/4485d18b888948439c034fbd5067496c is 50, key is test_row_0/B:col10/1731363500913/Put/seqid=0 2024-11-11T22:18:21,820 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111147c1a841b58142d391ca6a6abd6e7502_76df2be919cd8ff2500e96d76dee493a store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:21,822 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111147c1a841b58142d391ca6a6abd6e7502_76df2be919cd8ff2500e96d76dee493a, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:21,822 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111147c1a841b58142d391ca6a6abd6e7502_76df2be919cd8ff2500e96d76dee493a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:21,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742252_1428 (size=13051) 2024-11-11T22:18:21,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742253_1429 (size=4469) 2024-11-11T22:18:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T22:18:21,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T22:18:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:21,896 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:18:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:21,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f1dfb7b93cfa49ba932ef9947a7a86d4_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363500946/Put/seqid=0 2024-11-11T22:18:21,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742254_1430 (size=12454) 2024-11-11T22:18:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T22:18:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:22,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. as already flushing 2024-11-11T22:18:22,122 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-11-11T22:18:22,122 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-11T22:18:22,122 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:22,122 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:22,122 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:22,122 INFO [Thread-1543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1543. 2024-11-11T22:18:22,122 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:22,122 DEBUG [Thread-1543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:22,122 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:22,122 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:22,122 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:22,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:22,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:22,122 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:22,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:22,141 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-11T22:18:22,141 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:22,142 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:22,142 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:22,142 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:22,231 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/4485d18b888948439c034fbd5067496c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/4485d18b888948439c034fbd5067496c 2024-11-11T22:18:22,234 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#A#compaction#361 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:22,235 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/bdbe118e66b24f1ab593745675206e69 is 175, key is test_row_0/A:col10/1731363500913/Put/seqid=0 2024-11-11T22:18:22,239 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/B of 76df2be919cd8ff2500e96d76dee493a into 4485d18b888948439c034fbd5067496c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:22,239 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:22,239 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/B, priority=13, startTime=1731363501808; duration=0sec 2024-11-11T22:18:22,239 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:22,239 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:B 2024-11-11T22:18:22,239 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:22,240 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:22,240 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 76df2be919cd8ff2500e96d76dee493a/C is initiating minor compaction (all files) 2024-11-11T22:18:22,240 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76df2be919cd8ff2500e96d76dee493a/C in TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:22,240 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2e40fea53a514f20b820908d7bb06ceb, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4628e3fa4881429d983db2aba75ef099, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/875ffca3ede04a168908d1359585d4e2] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp, totalSize=36.7 K 2024-11-11T22:18:22,241 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e40fea53a514f20b820908d7bb06ceb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731363496817 2024-11-11T22:18:22,241 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4628e3fa4881429d983db2aba75ef099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731363499078 2024-11-11T22:18:22,241 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 875ffca3ede04a168908d1359585d4e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731363500270 2024-11-11T22:18:22,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742255_1431 (size=32005) 2024-11-11T22:18:22,251 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76df2be919cd8ff2500e96d76dee493a#C#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:22,251 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/2a33f74c4e1045afafbcbc655a175777 is 50, key is test_row_0/C:col10/1731363500913/Put/seqid=0 2024-11-11T22:18:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742256_1432 (size=13051) 2024-11-11T22:18:22,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:22,315 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f1dfb7b93cfa49ba932ef9947a7a86d4_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f1dfb7b93cfa49ba932ef9947a7a86d4_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/64bf5e2e8dec44589fb5dc2cf3059a24, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/64bf5e2e8dec44589fb5dc2cf3059a24 is 175, key is test_row_0/A:col10/1731363500946/Put/seqid=0 2024-11-11T22:18:22,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742257_1433 (size=31255) 2024-11-11T22:18:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T22:18:22,654 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/bdbe118e66b24f1ab593745675206e69 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/bdbe118e66b24f1ab593745675206e69 2024-11-11T22:18:22,658 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/2a33f74c4e1045afafbcbc655a175777 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2a33f74c4e1045afafbcbc655a175777 2024-11-11T22:18:22,661 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/A of 76df2be919cd8ff2500e96d76dee493a into bdbe118e66b24f1ab593745675206e69(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:22,661 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:22,661 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/A, priority=13, startTime=1731363501808; duration=0sec 2024-11-11T22:18:22,662 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:22,662 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:A 2024-11-11T22:18:22,662 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76df2be919cd8ff2500e96d76dee493a/C of 76df2be919cd8ff2500e96d76dee493a into 2a33f74c4e1045afafbcbc655a175777(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:22,663 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:22,663 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., storeName=76df2be919cd8ff2500e96d76dee493a/C, priority=13, startTime=1731363501808; duration=0sec 2024-11-11T22:18:22,663 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:22,663 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76df2be919cd8ff2500e96d76dee493a:C 2024-11-11T22:18:22,726 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/64bf5e2e8dec44589fb5dc2cf3059a24 2024-11-11T22:18:22,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/d1cdc3e43ee541eda3ee87fc603eaa64 is 50, key is test_row_0/B:col10/1731363500946/Put/seqid=0 2024-11-11T22:18:22,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742258_1434 (size=12301) 2024-11-11T22:18:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T22:18:23,139 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/d1cdc3e43ee541eda3ee87fc603eaa64 2024-11-11T22:18:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/b2a5a530130547508c0e61bf5e53970d is 50, key is test_row_0/C:col10/1731363500946/Put/seqid=0 2024-11-11T22:18:23,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742259_1435 (size=12301) 2024-11-11T22:18:23,331 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-11T22:18:23,331 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:23,331 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:23,331 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:23,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:23,549 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/b2a5a530130547508c0e61bf5e53970d 2024-11-11T22:18:23,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/64bf5e2e8dec44589fb5dc2cf3059a24 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/64bf5e2e8dec44589fb5dc2cf3059a24 2024-11-11T22:18:23,556 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/64bf5e2e8dec44589fb5dc2cf3059a24, entries=150, sequenceid=373, filesize=30.5 K 2024-11-11T22:18:23,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/d1cdc3e43ee541eda3ee87fc603eaa64 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d1cdc3e43ee541eda3ee87fc603eaa64 2024-11-11T22:18:23,561 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d1cdc3e43ee541eda3ee87fc603eaa64, entries=150, sequenceid=373, filesize=12.0 K 2024-11-11T22:18:23,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/b2a5a530130547508c0e61bf5e53970d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2a5a530130547508c0e61bf5e53970d 2024-11-11T22:18:23,565 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2a5a530130547508c0e61bf5e53970d, entries=150, sequenceid=373, filesize=12.0 K 2024-11-11T22:18:23,599 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=33.54 KB/34350 for 76df2be919cd8ff2500e96d76dee493a in 1703ms, sequenceid=373, compaction requested=false 2024-11-11T22:18:23,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 76df2be919cd8ff2500e96d76dee493a: 2024-11-11T22:18:23,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:23,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T22:18:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T22:18:23,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T22:18:23,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8580 sec 2024-11-11T22:18:23,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8630 sec 2024-11-11T22:18:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T22:18:23,899 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1080 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3238 rows 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3256 rows 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1085 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3251 rows 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1081 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3235 rows 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1082 2024-11-11T22:18:23,900 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3242 rows 2024-11-11T22:18:23,900 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:18:23,900 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:18:23,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:23,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:23,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:23,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:18:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T22:18:23,904 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363503904"}]},"ts":"1731363503904"} 2024-11-11T22:18:23,905 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:18:23,905 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:18:23,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:18:23,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, UNASSIGN}] 2024-11-11T22:18:23,908 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, UNASSIGN 2024-11-11T22:18:23,909 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=76df2be919cd8ff2500e96d76dee493a, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:23,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, UNASSIGN because future has completed 2024-11-11T22:18:23,910 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:18:23,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:18:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T22:18:24,063 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 76df2be919cd8ff2500e96d76dee493a, disabling compactions & flushes 2024-11-11T22:18:24,063 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. after waiting 0 ms 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:24,063 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 76df2be919cd8ff2500e96d76dee493a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=A 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=B 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76df2be919cd8ff2500e96d76dee493a, store=C 2024-11-11T22:18:24,063 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:24,069 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111398a6d5b7c7489eb757d96b92b30e80_76df2be919cd8ff2500e96d76dee493a is 50, key is test_row_0/A:col10/1731363502140/Put/seqid=0 2024-11-11T22:18:24,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742260_1436 (size=12454) 2024-11-11T22:18:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T22:18:24,473 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:24,477 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111398a6d5b7c7489eb757d96b92b30e80_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111398a6d5b7c7489eb757d96b92b30e80_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:24,478 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1c658181cd8147cfb22c41c3fc6bd43d, store: [table=TestAcidGuarantees family=A region=76df2be919cd8ff2500e96d76dee493a] 2024-11-11T22:18:24,478 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1c658181cd8147cfb22c41c3fc6bd43d is 175, key is test_row_0/A:col10/1731363502140/Put/seqid=0 2024-11-11T22:18:24,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742261_1437 (size=31255) 2024-11-11T22:18:24,484 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1c658181cd8147cfb22c41c3fc6bd43d 2024-11-11T22:18:24,491 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/61bf715686bc465faaa3048ce5bc9411 is 50, key is test_row_0/B:col10/1731363502140/Put/seqid=0 2024-11-11T22:18:24,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742262_1438 (size=12301) 2024-11-11T22:18:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T22:18:24,895 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/61bf715686bc465faaa3048ce5bc9411 2024-11-11T22:18:24,903 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/8dcaae3b96424b3db34bf6b8f4851018 is 50, key is test_row_0/C:col10/1731363502140/Put/seqid=0 2024-11-11T22:18:24,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742263_1439 (size=12301) 2024-11-11T22:18:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T22:18:25,308 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/8dcaae3b96424b3db34bf6b8f4851018 2024-11-11T22:18:25,311 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/A/1c658181cd8147cfb22c41c3fc6bd43d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1c658181cd8147cfb22c41c3fc6bd43d 2024-11-11T22:18:25,315 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1c658181cd8147cfb22c41c3fc6bd43d, entries=150, sequenceid=384, filesize=30.5 K 2024-11-11T22:18:25,315 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/B/61bf715686bc465faaa3048ce5bc9411 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/61bf715686bc465faaa3048ce5bc9411 2024-11-11T22:18:25,318 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/61bf715686bc465faaa3048ce5bc9411, entries=150, sequenceid=384, filesize=12.0 K 2024-11-11T22:18:25,319 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/.tmp/C/8dcaae3b96424b3db34bf6b8f4851018 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/8dcaae3b96424b3db34bf6b8f4851018 2024-11-11T22:18:25,321 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/8dcaae3b96424b3db34bf6b8f4851018, entries=150, sequenceid=384, filesize=12.0 K 2024-11-11T22:18:25,322 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 76df2be919cd8ff2500e96d76dee493a in 1259ms, sequenceid=384, compaction requested=true 2024-11-11T22:18:25,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27c85e0d3fb44f2d878bf1d33175b10e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1bf5cd6d0fa24e32b7c24402e441f596, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/12340b8e65d24068b62d1dfcbf92058f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4497f51b77b645ff959bc6f737d7c662, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/11bc83e228834e7faa015b80bdd95841, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ede11b12a8e14690a9e401479195fd3a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3d0abc79c2974d808e593abfc2458402, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ba8d322c097246c28cb63bcf80df993e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/67c3f235087e411cb3b5052a87fc23b7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/6a8ed7965ed34c1f9672df55e58f620b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/79a628a1dea34923ac3aef5471452442, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27b41abc7b2d4b98a12c94d99fac09fc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab378b70e5e146c49788ca956655f67e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4734a9cdb5dd417cb142841af5f38e1d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/9e226c052ce14a30aff2742effaedd9d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1d37221ac9bf48108a0900496f534dcf, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3b34721929934573b87eb488ecae2c78, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab9ca37023014f409eee9373964ca48e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/32f5aeea2d18428ba77b36b9e4d94d35, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/a8e5d6d323db45c49510e6931e0097a3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/d8996bbbbe6c48459586925a9e954e72, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/f22bc8467e2b477eb07b0959e81c0763] to archive 2024-11-11T22:18:25,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:18:25,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27c85e0d3fb44f2d878bf1d33175b10e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27c85e0d3fb44f2d878bf1d33175b10e 2024-11-11T22:18:25,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1bf5cd6d0fa24e32b7c24402e441f596 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1bf5cd6d0fa24e32b7c24402e441f596 2024-11-11T22:18:25,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/12340b8e65d24068b62d1dfcbf92058f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/12340b8e65d24068b62d1dfcbf92058f 2024-11-11T22:18:25,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4497f51b77b645ff959bc6f737d7c662 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4497f51b77b645ff959bc6f737d7c662 2024-11-11T22:18:25,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/11bc83e228834e7faa015b80bdd95841 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/11bc83e228834e7faa015b80bdd95841 2024-11-11T22:18:25,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ede11b12a8e14690a9e401479195fd3a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ede11b12a8e14690a9e401479195fd3a 2024-11-11T22:18:25,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3d0abc79c2974d808e593abfc2458402 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3d0abc79c2974d808e593abfc2458402 2024-11-11T22:18:25,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ba8d322c097246c28cb63bcf80df993e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ba8d322c097246c28cb63bcf80df993e 2024-11-11T22:18:25,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/67c3f235087e411cb3b5052a87fc23b7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/67c3f235087e411cb3b5052a87fc23b7 2024-11-11T22:18:25,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/6a8ed7965ed34c1f9672df55e58f620b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/6a8ed7965ed34c1f9672df55e58f620b 2024-11-11T22:18:25,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/79a628a1dea34923ac3aef5471452442 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/79a628a1dea34923ac3aef5471452442 2024-11-11T22:18:25,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27b41abc7b2d4b98a12c94d99fac09fc to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/27b41abc7b2d4b98a12c94d99fac09fc 2024-11-11T22:18:25,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab378b70e5e146c49788ca956655f67e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab378b70e5e146c49788ca956655f67e 2024-11-11T22:18:25,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4734a9cdb5dd417cb142841af5f38e1d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/4734a9cdb5dd417cb142841af5f38e1d 2024-11-11T22:18:25,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/9e226c052ce14a30aff2742effaedd9d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/9e226c052ce14a30aff2742effaedd9d 2024-11-11T22:18:25,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1d37221ac9bf48108a0900496f534dcf to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1d37221ac9bf48108a0900496f534dcf 2024-11-11T22:18:25,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3b34721929934573b87eb488ecae2c78 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/3b34721929934573b87eb488ecae2c78 2024-11-11T22:18:25,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab9ca37023014f409eee9373964ca48e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/ab9ca37023014f409eee9373964ca48e 2024-11-11T22:18:25,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/32f5aeea2d18428ba77b36b9e4d94d35 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/32f5aeea2d18428ba77b36b9e4d94d35 2024-11-11T22:18:25,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/a8e5d6d323db45c49510e6931e0097a3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/a8e5d6d323db45c49510e6931e0097a3 2024-11-11T22:18:25,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/d8996bbbbe6c48459586925a9e954e72 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/d8996bbbbe6c48459586925a9e954e72 2024-11-11T22:18:25,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/f22bc8467e2b477eb07b0959e81c0763 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/f22bc8467e2b477eb07b0959e81c0763 2024-11-11T22:18:25,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d4c8349d1d054287a31cbc9b59bc2f4c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1d4a7efa2e2c4694871a5273bdf2fb23, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/726282ed02514730bdebc66ac4c15473, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/86dbd244b11945dd9b8982bb9b013fe3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1fb9a2126e5146b1898566174ef23f91, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ca659c9eb90b40a2a62cc8250c7605ec, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3321b41534aa4fd1a128a76a69504554, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/15d343c6cd2545cda70c8c01d26e9d35, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/a40b1eacb15f4c5482105ac0c05850b1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bdb07940bd7a42808da0fff6fbe6beaa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/194c26aa99c8473bac8fc659730353ec, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/958748fb498a4ab99920f8f838f9d58a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/6b6e32a6c2ce4c5fbb1ff524db56d66f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/09c9ee7cfc644323b9f7a72d4b360ea4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/43c60a6987a04fa7ad78a2094c16ce0f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ccebb84b2a05438cbe44468d997db72d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bf6b3f2826a64b0989d4ee43b52b2e45, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/655ebbd99f9f4c438aa59a4d0bf92fa0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3096c78cef26408b8bc0c07b41be9e94, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9f641bea48d84a61a6c8fe8162753c6c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9d171b98a3c2456487ac426d0d433133, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/7fe337655d804f128a11957842fc4510] to archive 2024-11-11T22:18:25,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:18:25,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d4c8349d1d054287a31cbc9b59bc2f4c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d4c8349d1d054287a31cbc9b59bc2f4c 2024-11-11T22:18:25,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1d4a7efa2e2c4694871a5273bdf2fb23 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1d4a7efa2e2c4694871a5273bdf2fb23 2024-11-11T22:18:25,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/726282ed02514730bdebc66ac4c15473 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/726282ed02514730bdebc66ac4c15473 2024-11-11T22:18:25,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/86dbd244b11945dd9b8982bb9b013fe3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/86dbd244b11945dd9b8982bb9b013fe3 2024-11-11T22:18:25,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1fb9a2126e5146b1898566174ef23f91 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/1fb9a2126e5146b1898566174ef23f91 2024-11-11T22:18:25,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ca659c9eb90b40a2a62cc8250c7605ec to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ca659c9eb90b40a2a62cc8250c7605ec 2024-11-11T22:18:25,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3321b41534aa4fd1a128a76a69504554 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3321b41534aa4fd1a128a76a69504554 2024-11-11T22:18:25,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/15d343c6cd2545cda70c8c01d26e9d35 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/15d343c6cd2545cda70c8c01d26e9d35 2024-11-11T22:18:25,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/a40b1eacb15f4c5482105ac0c05850b1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/a40b1eacb15f4c5482105ac0c05850b1 2024-11-11T22:18:25,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bdb07940bd7a42808da0fff6fbe6beaa to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bdb07940bd7a42808da0fff6fbe6beaa 2024-11-11T22:18:25,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/194c26aa99c8473bac8fc659730353ec to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/194c26aa99c8473bac8fc659730353ec 2024-11-11T22:18:25,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/958748fb498a4ab99920f8f838f9d58a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/958748fb498a4ab99920f8f838f9d58a 2024-11-11T22:18:25,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/6b6e32a6c2ce4c5fbb1ff524db56d66f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/6b6e32a6c2ce4c5fbb1ff524db56d66f 2024-11-11T22:18:25,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/09c9ee7cfc644323b9f7a72d4b360ea4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/09c9ee7cfc644323b9f7a72d4b360ea4 2024-11-11T22:18:25,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/43c60a6987a04fa7ad78a2094c16ce0f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/43c60a6987a04fa7ad78a2094c16ce0f 2024-11-11T22:18:25,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ccebb84b2a05438cbe44468d997db72d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/ccebb84b2a05438cbe44468d997db72d 2024-11-11T22:18:25,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bf6b3f2826a64b0989d4ee43b52b2e45 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/bf6b3f2826a64b0989d4ee43b52b2e45 2024-11-11T22:18:25,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/655ebbd99f9f4c438aa59a4d0bf92fa0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/655ebbd99f9f4c438aa59a4d0bf92fa0 2024-11-11T22:18:25,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3096c78cef26408b8bc0c07b41be9e94 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/3096c78cef26408b8bc0c07b41be9e94 2024-11-11T22:18:25,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9f641bea48d84a61a6c8fe8162753c6c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9f641bea48d84a61a6c8fe8162753c6c 2024-11-11T22:18:25,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9d171b98a3c2456487ac426d0d433133 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/9d171b98a3c2456487ac426d0d433133 2024-11-11T22:18:25,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/7fe337655d804f128a11957842fc4510 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/7fe337655d804f128a11957842fc4510 2024-11-11T22:18:25,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/ee107ab4a20044b49fac9fef164e9e59, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2b678e3b85d480095ab6f185e4e005b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/853af3561f304e3f89a7a1406719a5d0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/9aec473cf43b4fde807fb86bf6e92e36, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/035d50318d954df8b41133fd32c5f462, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/278b037d154941c5a1228e46ff76cb28, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/12c45f8de1fe443c8dd7c36ac285584c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fc00db0920ce4fcdbe5acce32c65c5c5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e106382a03784649a8afce52d080b7ad, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/38fdcb06b2ba49dd9a49e919a85914d8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/80e0ff262f824669b65857e84cc2391f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/1d3fc00522d344dcbb954d9105d4cc1d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e4c74e1a97de43fbae78c2b248127274, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4d5993e854854bd3ae48f8c8efe4e1f8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2f6e849e6e4e4dabb238692790b0ce0c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/6a68ccb5f05c4852b2dfb3b346dfdf98, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/06411c9c2dfc434bbb52a32088b7fef3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/383b48dd51284a169b8bda81679f717e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2e40fea53a514f20b820908d7bb06ceb, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fcb043fc521841329ae6eb35c40be398, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4628e3fa4881429d983db2aba75ef099, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/875ffca3ede04a168908d1359585d4e2] to archive 2024-11-11T22:18:25,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:18:25,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/ee107ab4a20044b49fac9fef164e9e59 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/ee107ab4a20044b49fac9fef164e9e59 2024-11-11T22:18:25,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2b678e3b85d480095ab6f185e4e005b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2b678e3b85d480095ab6f185e4e005b 2024-11-11T22:18:25,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/853af3561f304e3f89a7a1406719a5d0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/853af3561f304e3f89a7a1406719a5d0 2024-11-11T22:18:25,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/9aec473cf43b4fde807fb86bf6e92e36 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/9aec473cf43b4fde807fb86bf6e92e36 2024-11-11T22:18:25,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/035d50318d954df8b41133fd32c5f462 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/035d50318d954df8b41133fd32c5f462 2024-11-11T22:18:25,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/278b037d154941c5a1228e46ff76cb28 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/278b037d154941c5a1228e46ff76cb28 2024-11-11T22:18:25,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/12c45f8de1fe443c8dd7c36ac285584c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/12c45f8de1fe443c8dd7c36ac285584c 2024-11-11T22:18:25,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fc00db0920ce4fcdbe5acce32c65c5c5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fc00db0920ce4fcdbe5acce32c65c5c5 2024-11-11T22:18:25,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e106382a03784649a8afce52d080b7ad to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e106382a03784649a8afce52d080b7ad 2024-11-11T22:18:25,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/38fdcb06b2ba49dd9a49e919a85914d8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/38fdcb06b2ba49dd9a49e919a85914d8 2024-11-11T22:18:25,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/80e0ff262f824669b65857e84cc2391f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/80e0ff262f824669b65857e84cc2391f 2024-11-11T22:18:25,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/1d3fc00522d344dcbb954d9105d4cc1d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/1d3fc00522d344dcbb954d9105d4cc1d 2024-11-11T22:18:25,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e4c74e1a97de43fbae78c2b248127274 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/e4c74e1a97de43fbae78c2b248127274 2024-11-11T22:18:25,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4d5993e854854bd3ae48f8c8efe4e1f8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4d5993e854854bd3ae48f8c8efe4e1f8 2024-11-11T22:18:25,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2f6e849e6e4e4dabb238692790b0ce0c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2f6e849e6e4e4dabb238692790b0ce0c 2024-11-11T22:18:25,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/6a68ccb5f05c4852b2dfb3b346dfdf98 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/6a68ccb5f05c4852b2dfb3b346dfdf98 2024-11-11T22:18:25,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/06411c9c2dfc434bbb52a32088b7fef3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/06411c9c2dfc434bbb52a32088b7fef3 2024-11-11T22:18:25,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/383b48dd51284a169b8bda81679f717e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/383b48dd51284a169b8bda81679f717e 2024-11-11T22:18:25,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2e40fea53a514f20b820908d7bb06ceb to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2e40fea53a514f20b820908d7bb06ceb 2024-11-11T22:18:25,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fcb043fc521841329ae6eb35c40be398 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/fcb043fc521841329ae6eb35c40be398 2024-11-11T22:18:25,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4628e3fa4881429d983db2aba75ef099 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/4628e3fa4881429d983db2aba75ef099 2024-11-11T22:18:25,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/875ffca3ede04a168908d1359585d4e2 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/875ffca3ede04a168908d1359585d4e2 2024-11-11T22:18:25,402 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/recovered.edits/387.seqid, newMaxSeqId=387, maxSeqId=4 2024-11-11T22:18:25,403 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. 2024-11-11T22:18:25,403 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 76df2be919cd8ff2500e96d76dee493a: Waiting for close lock at 1731363504063Running coprocessor pre-close hooks at 1731363504063Disabling compacts and flushes for region at 1731363504063Disabling writes for close at 1731363504063Obtaining lock to block concurrent updates at 1731363504063Preparing flush snapshotting stores in 76df2be919cd8ff2500e96d76dee493a at 1731363504063Finished memstore snapshotting TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731363504064 (+1 ms)Flushing stores of TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a. at 1731363504064Flushing 76df2be919cd8ff2500e96d76dee493a/A: creating writer at 1731363504064Flushing mob file 76df2be919cd8ff2500e96d76dee493a/A: appending metadata at 1731363504069 (+5 ms)Flushing mob file 76df2be919cd8ff2500e96d76dee493a/A: closing flushed file at 1731363504069Flushing 76df2be919cd8ff2500e96d76dee493a/A: appending metadata at 1731363504478 (+409 ms)Flushing 76df2be919cd8ff2500e96d76dee493a/A: closing flushed file at 1731363504478Flushing 76df2be919cd8ff2500e96d76dee493a/B: creating writer at 1731363504487 (+9 ms)Flushing 76df2be919cd8ff2500e96d76dee493a/B: appending metadata at 1731363504491 (+4 ms)Flushing 76df2be919cd8ff2500e96d76dee493a/B: closing flushed file at 1731363504491Flushing 76df2be919cd8ff2500e96d76dee493a/C: creating writer at 1731363504899 (+408 ms)Flushing 76df2be919cd8ff2500e96d76dee493a/C: appending metadata at 1731363504902 (+3 ms)Flushing 76df2be919cd8ff2500e96d76dee493a/C: closing flushed file at 1731363504903 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2614da4c: reopening flushed file at 1731363505310 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39571700: reopening flushed file at 1731363505315 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d9aaa1: reopening flushed file at 1731363505318 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 76df2be919cd8ff2500e96d76dee493a in 1259ms, sequenceid=384, compaction requested=true at 1731363505322 (+4 ms)Writing region close event to WAL at 1731363505399 (+77 ms)Running coprocessor post-close hooks at 1731363505403 (+4 ms)Closed at 1731363505403 2024-11-11T22:18:25,405 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:25,405 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=76df2be919cd8ff2500e96d76dee493a, regionState=CLOSED 2024-11-11T22:18:25,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:18:25,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T22:18:25,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 76df2be919cd8ff2500e96d76dee493a, server=8d82b89dd27d,36911,1731363387997 in 1.4970 sec 2024-11-11T22:18:25,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-11T22:18:25,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76df2be919cd8ff2500e96d76dee493a, UNASSIGN in 1.5020 sec 2024-11-11T22:18:25,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T22:18:25,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5050 sec 2024-11-11T22:18:25,415 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363505415"}]},"ts":"1731363505415"} 2024-11-11T22:18:25,417 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:18:25,417 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:18:25,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5170 sec 2024-11-11T22:18:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T22:18:26,049 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:26,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:18:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:26,051 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:18:26,052 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:26,055 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,057 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/recovered.edits] 2024-11-11T22:18:26,060 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1c658181cd8147cfb22c41c3fc6bd43d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/1c658181cd8147cfb22c41c3fc6bd43d 2024-11-11T22:18:26,061 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/64bf5e2e8dec44589fb5dc2cf3059a24 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/64bf5e2e8dec44589fb5dc2cf3059a24 2024-11-11T22:18:26,062 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/bdbe118e66b24f1ab593745675206e69 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/A/bdbe118e66b24f1ab593745675206e69 2024-11-11T22:18:26,065 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/4485d18b888948439c034fbd5067496c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/4485d18b888948439c034fbd5067496c 2024-11-11T22:18:26,066 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/61bf715686bc465faaa3048ce5bc9411 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/61bf715686bc465faaa3048ce5bc9411 2024-11-11T22:18:26,066 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d1cdc3e43ee541eda3ee87fc603eaa64 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/B/d1cdc3e43ee541eda3ee87fc603eaa64 2024-11-11T22:18:26,068 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2a33f74c4e1045afafbcbc655a175777 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/2a33f74c4e1045afafbcbc655a175777 2024-11-11T22:18:26,069 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/8dcaae3b96424b3db34bf6b8f4851018 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/8dcaae3b96424b3db34bf6b8f4851018 2024-11-11T22:18:26,069 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2a5a530130547508c0e61bf5e53970d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/C/b2a5a530130547508c0e61bf5e53970d 2024-11-11T22:18:26,074 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/recovered.edits/387.seqid to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a/recovered.edits/387.seqid 2024-11-11T22:18:26,074 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,075 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:18:26,075 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:18:26,075 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T22:18:26,077 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106f65cfc6d204c0bba292dd3959c861a_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106f65cfc6d204c0bba292dd3959c861a_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,079 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a154c05db85437aaef1823ca9fb584d_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a154c05db85437aaef1823ca9fb584d_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,080 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110f3f9fe275e4ddba2c64243a8478ede_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111110f3f9fe275e4ddba2c64243a8478ede_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,081 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111398a6d5b7c7489eb757d96b92b30e80_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111398a6d5b7c7489eb757d96b92b30e80_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,082 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117cc2dd20f6f4b05a3ba5ae3af9e19b5_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111117cc2dd20f6f4b05a3ba5ae3af9e19b5_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,083 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111cb987e985f74a8b9e7bec07978346bb_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111cb987e985f74a8b9e7bec07978346bb_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,083 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111d50bc38e31a4ac6b26b5122d775a17e_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111d50bc38e31a4ac6b26b5122d775a17e_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,084 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111126448a7546b84d36af0b10f975b07d1f_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111126448a7546b84d36af0b10f975b07d1f_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,085 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112fec6aeb70604dc980bba5de9fbc1a6b_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112fec6aeb70604dc980bba5de9fbc1a6b_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,085 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111138cd8b73bb7f4eb68dac1dd76437d003_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111138cd8b73bb7f4eb68dac1dd76437d003_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,086 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146babdb55db24554985743e11ec2ac6b_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146babdb55db24554985743e11ec2ac6b_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,087 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a40640eab824a12965051adefa6bd03_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115a40640eab824a12965051adefa6bd03_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,087 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111827195a2bf954e6c900c74df36ef88fa_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111827195a2bf954e6c900c74df36ef88fa_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,088 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111973c6f9c9a5942b0baef720caefade3e_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111973c6f9c9a5942b0baef720caefade3e_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,089 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a78476b05399484592d49b048d85f66b_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a78476b05399484592d49b048d85f66b_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,090 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b6ba78c0858a4f909e8b7d763861bd56_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b6ba78c0858a4f909e8b7d763861bd56_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,090 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc0970de3017496a93ec49ca44393d48_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111dc0970de3017496a93ec49ca44393d48_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,091 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f1dfb7b93cfa49ba932ef9947a7a86d4_76df2be919cd8ff2500e96d76dee493a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f1dfb7b93cfa49ba932ef9947a7a86d4_76df2be919cd8ff2500e96d76dee493a 2024-11-11T22:18:26,091 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:18:26,093 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:26,095 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:18:26,096 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:18:26,097 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:26,097 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:18:26,097 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731363506097"}]},"ts":"9223372036854775807"} 2024-11-11T22:18:26,099 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:18:26,099 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 76df2be919cd8ff2500e96d76dee493a, NAME => 'TestAcidGuarantees,,1731363478679.76df2be919cd8ff2500e96d76dee493a.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:18:26,099 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:18:26,099 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731363506099"}]},"ts":"9223372036854775807"} 2024-11-11T22:18:26,100 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:18:26,101 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:26,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-11T22:18:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T22:18:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:18:26,162 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:26,168 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 251), OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=658 (was 784), ProcessCount=11 (was 11), AvailableMemoryMB=1624 (was 1011) - AvailableMemoryMB LEAK? - 2024-11-11T22:18:26,176 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=658, ProcessCount=11, AvailableMemoryMB=1623 2024-11-11T22:18:26,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:18:26,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:18:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:26,179 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:18:26,179 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:26,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-11T22:18:26,180 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:18:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T22:18:26,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742264_1440 (size=1061) 2024-11-11T22:18:26,250 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:18:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T22:18:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T22:18:26,586 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e 2024-11-11T22:18:26,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742265_1441 (size=53) 2024-11-11T22:18:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T22:18:26,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:18:26,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 20d39f0a33cd5786c48995acee2197ce, disabling compactions & flushes 2024-11-11T22:18:26,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:26,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:26,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. after waiting 0 ms 2024-11-11T22:18:26,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:26,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:26,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 20d39f0a33cd5786c48995acee2197ce: Waiting for close lock at 1731363506992Disabling compacts and flushes for region at 1731363506992Disabling writes for close at 1731363506992Writing region close event to WAL at 1731363506992Closed at 1731363506992 2024-11-11T22:18:26,994 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:18:26,994 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363506994"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363506994"}]},"ts":"1731363506994"} 2024-11-11T22:18:26,996 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:18:26,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:18:26,998 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363506998"}]},"ts":"1731363506998"} 2024-11-11T22:18:27,001 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:18:27,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, ASSIGN}] 2024-11-11T22:18:27,003 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, ASSIGN 2024-11-11T22:18:27,004 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, ASSIGN; state=OFFLINE, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=false 2024-11-11T22:18:27,154 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=20d39f0a33cd5786c48995acee2197ce, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:27,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, ASSIGN because future has completed 2024-11-11T22:18:27,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:18:27,312 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:27,312 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:18:27,312 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,312 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:18:27,312 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,312 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,314 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,315 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:27,315 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20d39f0a33cd5786c48995acee2197ce columnFamilyName A 2024-11-11T22:18:27,315 DEBUG [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:27,315 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.HStore(327): Store=20d39f0a33cd5786c48995acee2197ce/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:27,315 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,316 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:27,316 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20d39f0a33cd5786c48995acee2197ce columnFamilyName B 2024-11-11T22:18:27,317 DEBUG [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:27,317 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.HStore(327): Store=20d39f0a33cd5786c48995acee2197ce/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:27,317 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,318 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:27,318 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20d39f0a33cd5786c48995acee2197ce columnFamilyName C 2024-11-11T22:18:27,318 DEBUG [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:27,318 INFO [StoreOpener-20d39f0a33cd5786c48995acee2197ce-1 {}] regionserver.HStore(327): Store=20d39f0a33cd5786c48995acee2197ce/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:27,318 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:27,318 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,319 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,319 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T22:18:27,320 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,320 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,320 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:18:27,321 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,323 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:18:27,323 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 20d39f0a33cd5786c48995acee2197ce; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73340041, jitterRate=0.09285177290439606}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:18:27,323 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:27,324 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 20d39f0a33cd5786c48995acee2197ce: Running coprocessor pre-open hook at 1731363507312Writing region info on filesystem at 1731363507312Initializing all the Stores at 1731363507313 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363507313Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363507313Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363507313Cleaning up temporary data from old regions at 1731363507320 (+7 ms)Running coprocessor post-open hooks at 1731363507323 (+3 ms)Region opened successfully at 1731363507324 (+1 ms) 2024-11-11T22:18:27,325 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., pid=116, masterSystemTime=1731363507309 2024-11-11T22:18:27,327 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:27,327 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:27,327 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=20d39f0a33cd5786c48995acee2197ce, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:27,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:18:27,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T22:18:27,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 in 172 msec 2024-11-11T22:18:27,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-11T22:18:27,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, ASSIGN in 330 msec 2024-11-11T22:18:27,333 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:18:27,333 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363507333"}]},"ts":"1731363507333"} 2024-11-11T22:18:27,335 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:18:27,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:18:27,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1580 sec 2024-11-11T22:18:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-11T22:18:28,339 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:28,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a68ca01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,341 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a79721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bab2b71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,344 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,346 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:18:28,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,41919,1731363387169 2024-11-11T22:18:28,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c78d264 2024-11-11T22:18:28,347 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:18:28,348 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:18:28,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc6145c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,350 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416b3ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d5e3ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323c960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,353 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbae6ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,354 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c5feca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bc71f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,356 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7623dc33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,358 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4959b217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab65de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,361 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a59eda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,362 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b4530b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b3d1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,363 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9da477, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,365 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@445ebf12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337c4d69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,368 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c44790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,370 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672c74df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b5c3343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,372 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@748f8d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,374 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf1ecc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92c812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,376 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@359dd5ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,378 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39a266bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a00d5a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,380 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcfc78b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,382 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1092eab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f5073a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:28,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:28,384 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dfe59a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:28,385 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1caae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:28,387 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,387 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,388 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,388 DEBUG [Thread-1984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,388 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,389 DEBUG [Thread-1980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,394 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,394 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,396 DEBUG [Thread-1986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:28,398 DEBUG [Thread-1999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:28,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-11T22:18:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:18:28,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:28,402 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,403 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:28,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2] 2024-11-11T22:18:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:28,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:18:28,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:28,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:28,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:28,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:28,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:28,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:28,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/906af6cd283a474b9d2d46f6c5a18444 is 50, key is test_row_0/A:col10/1731363508395/Put/seqid=0 2024-11-11T22:18:28,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742266_1442 (size=16681) 2024-11-11T22:18:28,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/906af6cd283a474b9d2d46f6c5a18444 2024-11-11T22:18:28,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731363508418, max timestamp=1731363508453Num uniques 859; 2024-11-11T22:18:28,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731363508418, max timestamp=1731363508453Num uniques 859; 2024-11-11T22:18:28,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731363508418, max timestamp=1731363508453Num uniques 859; 2024-11-11T22:18:28,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T22:18:28,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T22:18:28,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T22:18:28,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d3a1a684b5244469a89cf6409f141b05 is 50, key is test_row_0/B:col10/1731363508395/Put/seqid=0 2024-11-11T22:18:28,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742267_1443 (size=12001) 2024-11-11T22:18:28,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d3a1a684b5244469a89cf6409f141b05 2024-11-11T22:18:28,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363568507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:18:28,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363568517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363568518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/5d29adc6140e4a6aa81dd0fccb0c3eca is 50, key is test_row_0/C:col10/1731363508395/Put/seqid=0 2024-11-11T22:18:28,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363568519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363568519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:28,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:28,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:28,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:28,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742268_1444 (size=12001) 2024-11-11T22:18:28,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/5d29adc6140e4a6aa81dd0fccb0c3eca 2024-11-11T22:18:28,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/906af6cd283a474b9d2d46f6c5a18444 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/906af6cd283a474b9d2d46f6c5a18444 2024-11-11T22:18:28,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/906af6cd283a474b9d2d46f6c5a18444, entries=250, sequenceid=11, filesize=16.3 K 2024-11-11T22:18:28,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d3a1a684b5244469a89cf6409f141b05 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d3a1a684b5244469a89cf6409f141b05 2024-11-11T22:18:28,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d3a1a684b5244469a89cf6409f141b05, entries=150, sequenceid=11, filesize=11.7 K 2024-11-11T22:18:28,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/5d29adc6140e4a6aa81dd0fccb0c3eca as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/5d29adc6140e4a6aa81dd0fccb0c3eca 2024-11-11T22:18:28,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/5d29adc6140e4a6aa81dd0fccb0c3eca, entries=150, sequenceid=11, filesize=11.7 K 2024-11-11T22:18:28,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 20d39f0a33cd5786c48995acee2197ce in 196ms, sequenceid=11, compaction requested=false 2024-11-11T22:18:28,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:28,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-11T22:18:28,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:28,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:28,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:28,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:28,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:28,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:28,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/b067c61538a74e1bb770fdc2832c0454 is 50, key is test_row_0/A:col10/1731363508515/Put/seqid=0 2024-11-11T22:18:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363568639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363568639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363568647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363568652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363568654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742269_1445 (size=14341) 2024-11-11T22:18:28,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:28,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:28,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:28,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:28,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:18:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363568759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363568769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363568769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363568779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:28,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:28,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:28,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:28,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:28,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363568859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363568969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363568989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363568990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363568999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:18:29,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/b067c61538a74e1bb770fdc2832c0454 2024-11-11T22:18:29,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/65dfa58a494e4896be0491abbacdb801 is 50, key is test_row_0/B:col10/1731363508515/Put/seqid=0 2024-11-11T22:18:29,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742270_1446 (size=12001) 2024-11-11T22:18:29,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363569179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363569289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363569299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363569299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363569320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/65dfa58a494e4896be0491abbacdb801 2024-11-11T22:18:29,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/54172755e06f45b98f53b410d6dafdb5 is 50, key is test_row_0/C:col10/1731363508515/Put/seqid=0 2024-11-11T22:18:29,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742271_1447 (size=12001) 2024-11-11T22:18:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:18:29,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:29,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363569689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:29,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363569799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363569809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363569809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363569829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:29,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/54172755e06f45b98f53b410d6dafdb5 2024-11-11T22:18:29,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/b067c61538a74e1bb770fdc2832c0454 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/b067c61538a74e1bb770fdc2832c0454 2024-11-11T22:18:29,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/b067c61538a74e1bb770fdc2832c0454, entries=200, sequenceid=47, filesize=14.0 K 2024-11-11T22:18:29,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/65dfa58a494e4896be0491abbacdb801 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/65dfa58a494e4896be0491abbacdb801 2024-11-11T22:18:29,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/65dfa58a494e4896be0491abbacdb801, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T22:18:29,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/54172755e06f45b98f53b410d6dafdb5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/54172755e06f45b98f53b410d6dafdb5 2024-11-11T22:18:29,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/54172755e06f45b98f53b410d6dafdb5, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T22:18:29,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for 20d39f0a33cd5786c48995acee2197ce in 1282ms, sequenceid=47, compaction requested=false 2024-11-11T22:18:29,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:29,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T22:18:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:29,935 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T22:18:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/2b72077a3f074f879590ed064e4c875f is 50, key is test_row_1/A:col10/1731363508650/Put/seqid=0 2024-11-11T22:18:29,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742272_1448 (size=9657) 2024-11-11T22:18:29,980 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/2b72077a3f074f879590ed064e4c875f 2024-11-11T22:18:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/662331c4d072420bb530bbdcac85d3df is 50, key is test_row_1/B:col10/1731363508650/Put/seqid=0 2024-11-11T22:18:30,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742273_1449 (size=9657) 2024-11-11T22:18:30,405 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/662331c4d072420bb530bbdcac85d3df 2024-11-11T22:18:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/53fb5ee6d1d24f36a24a5c72bbd6239c is 50, key is test_row_1/C:col10/1731363508650/Put/seqid=0 2024-11-11T22:18:30,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742274_1450 (size=9657) 2024-11-11T22:18:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:18:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:30,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:30,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363510782, max timestamp=1731363510826Num uniques 859; 2024-11-11T22:18:30,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363510782, max timestamp=1731363510826Num uniques 859; 2024-11-11T22:18:30,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:30,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:30,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731363510782, max timestamp=1731363510826Num uniques 859; 2024-11-11T22:18:30,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T22:18:30,837 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/53fb5ee6d1d24f36a24a5c72bbd6239c 2024-11-11T22:18:30,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/2b72077a3f074f879590ed064e4c875f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b72077a3f074f879590ed064e4c875f 2024-11-11T22:18:30,845 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b72077a3f074f879590ed064e4c875f, entries=100, sequenceid=55, filesize=9.4 K 2024-11-11T22:18:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/662331c4d072420bb530bbdcac85d3df as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/662331c4d072420bb530bbdcac85d3df 2024-11-11T22:18:30,853 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/662331c4d072420bb530bbdcac85d3df, entries=100, sequenceid=55, filesize=9.4 K 2024-11-11T22:18:30,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/53fb5ee6d1d24f36a24a5c72bbd6239c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/53fb5ee6d1d24f36a24a5c72bbd6239c 2024-11-11T22:18:30,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/53fb5ee6d1d24f36a24a5c72bbd6239c, entries=100, sequenceid=55, filesize=9.4 K 2024-11-11T22:18:30,865 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=93.93 KB/96180 for 20d39f0a33cd5786c48995acee2197ce in 930ms, sequenceid=55, compaction requested=true 2024-11-11T22:18:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-11T22:18:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:30,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=100.63 KB heapSize=280.69 KB 2024-11-11T22:18:30,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:30,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-11T22:18:30,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1dbb96a3a98c412ea33b462fa596faf7 is 50, key is test_row_0/A:col10/1731363510859/Put/seqid=0 2024-11-11T22:18:30,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T22:18:30,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4650 sec 2024-11-11T22:18:30,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.4830 sec 2024-11-11T22:18:30,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:30,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363570894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:30,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:30,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742275_1451 (size=19021) 2024-11-11T22:18:30,922 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:18:30,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363570907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363570908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:30,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363570913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:30,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:30,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363570913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,025 WARN [regionserver/8d82b89dd27d:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-11T22:18:31,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363571024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363571040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363571040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363571040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363571040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363571249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363571249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363571250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363571259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363571259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1dbb96a3a98c412ea33b462fa596faf7 2024-11-11T22:18:31,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/74b449119c5b45c0aa928320ea85abb7 is 50, key is test_row_0/B:col10/1731363510859/Put/seqid=0 2024-11-11T22:18:31,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742276_1452 (size=12001) 2024-11-11T22:18:31,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363571569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363571570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363571570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363571579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363571579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:31,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/74b449119c5b45c0aa928320ea85abb7 2024-11-11T22:18:31,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/77fc97b10c5f4ce68cb2a8a38cbf5416 is 50, key is test_row_0/C:col10/1731363510859/Put/seqid=0 2024-11-11T22:18:31,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742277_1453 (size=12001) 2024-11-11T22:18:31,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/77fc97b10c5f4ce68cb2a8a38cbf5416 2024-11-11T22:18:31,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1dbb96a3a98c412ea33b462fa596faf7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1dbb96a3a98c412ea33b462fa596faf7 2024-11-11T22:18:31,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1dbb96a3a98c412ea33b462fa596faf7, entries=300, sequenceid=86, filesize=18.6 K 2024-11-11T22:18:31,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/74b449119c5b45c0aa928320ea85abb7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74b449119c5b45c0aa928320ea85abb7 2024-11-11T22:18:31,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74b449119c5b45c0aa928320ea85abb7, entries=150, sequenceid=86, filesize=11.7 K 2024-11-11T22:18:31,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/77fc97b10c5f4ce68cb2a8a38cbf5416 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/77fc97b10c5f4ce68cb2a8a38cbf5416 2024-11-11T22:18:31,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/77fc97b10c5f4ce68cb2a8a38cbf5416, entries=150, sequenceid=86, filesize=11.7 K 2024-11-11T22:18:31,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~280.64 KB/287376, currentSize=87.22 KB/89310 for 20d39f0a33cd5786c48995acee2197ce in 921ms, sequenceid=86, compaction requested=true 2024-11-11T22:18:31,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:31,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:31,786 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:18:31,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:31,786 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:18:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:31,787 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:18:31,788 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/B is initiating minor compaction (all files) 2024-11-11T22:18:31,788 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/B in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:31,788 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:18:31,788 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/A is initiating minor compaction (all files) 2024-11-11T22:18:31,788 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d3a1a684b5244469a89cf6409f141b05, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/65dfa58a494e4896be0491abbacdb801, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/662331c4d072420bb530bbdcac85d3df, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74b449119c5b45c0aa928320ea85abb7] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=44.6 K 2024-11-11T22:18:31,788 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/A in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:31,788 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/906af6cd283a474b9d2d46f6c5a18444, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/b067c61538a74e1bb770fdc2832c0454, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b72077a3f074f879590ed064e4c875f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1dbb96a3a98c412ea33b462fa596faf7] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=58.3 K 2024-11-11T22:18:31,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 906af6cd283a474b9d2d46f6c5a18444, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731363508395 2024-11-11T22:18:31,790 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d3a1a684b5244469a89cf6409f141b05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731363508395 2024-11-11T22:18:31,790 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 65dfa58a494e4896be0491abbacdb801, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731363508515 2024-11-11T22:18:31,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b067c61538a74e1bb770fdc2832c0454, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731363508515 2024-11-11T22:18:31,792 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b72077a3f074f879590ed064e4c875f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731363508650 2024-11-11T22:18:31,792 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dbb96a3a98c412ea33b462fa596faf7, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731363510841 2024-11-11T22:18:31,795 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 662331c4d072420bb530bbdcac85d3df, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731363508650 2024-11-11T22:18:31,795 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 74b449119c5b45c0aa928320ea85abb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731363510841 2024-11-11T22:18:31,810 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#A#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:31,810 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/3a25fa886b7248ab9b9068336f4cfeb7 is 50, key is test_row_0/A:col10/1731363510859/Put/seqid=0 2024-11-11T22:18:31,824 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#B#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:31,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742278_1454 (size=12139) 2024-11-11T22:18:31,824 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/dc585f97d6df4b12b73e42dff9850716 is 50, key is test_row_0/B:col10/1731363510859/Put/seqid=0 2024-11-11T22:18:31,829 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/3a25fa886b7248ab9b9068336f4cfeb7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/3a25fa886b7248ab9b9068336f4cfeb7 2024-11-11T22:18:31,832 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/A of 20d39f0a33cd5786c48995acee2197ce into 3a25fa886b7248ab9b9068336f4cfeb7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:31,833 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:31,833 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/A, priority=12, startTime=1731363511786; duration=0sec 2024-11-11T22:18:31,833 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:31,833 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:A 2024-11-11T22:18:31,833 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:18:31,835 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:18:31,835 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/C is initiating minor compaction (all files) 2024-11-11T22:18:31,835 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/C in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:31,835 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/5d29adc6140e4a6aa81dd0fccb0c3eca, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/54172755e06f45b98f53b410d6dafdb5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/53fb5ee6d1d24f36a24a5c72bbd6239c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/77fc97b10c5f4ce68cb2a8a38cbf5416] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=44.6 K 2024-11-11T22:18:31,836 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d29adc6140e4a6aa81dd0fccb0c3eca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731363508395 2024-11-11T22:18:31,836 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54172755e06f45b98f53b410d6dafdb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731363508515 2024-11-11T22:18:31,836 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53fb5ee6d1d24f36a24a5c72bbd6239c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731363508650 2024-11-11T22:18:31,836 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77fc97b10c5f4ce68cb2a8a38cbf5416, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731363510841 2024-11-11T22:18:31,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742279_1455 (size=12139) 2024-11-11T22:18:31,895 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/dc585f97d6df4b12b73e42dff9850716 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dc585f97d6df4b12b73e42dff9850716 2024-11-11T22:18:31,896 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#C#compaction#383 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:31,903 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/B of 20d39f0a33cd5786c48995acee2197ce into dc585f97d6df4b12b73e42dff9850716(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:31,903 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:31,903 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/B, priority=12, startTime=1731363511786; duration=0sec 2024-11-11T22:18:31,905 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:31,905 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:B 2024-11-11T22:18:31,905 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/4b4a404e9f2a4b40be292f0ad8d93807 is 50, key is test_row_0/C:col10/1731363510859/Put/seqid=0 2024-11-11T22:18:31,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742280_1456 (size=12139) 2024-11-11T22:18:31,931 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/4b4a404e9f2a4b40be292f0ad8d93807 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4b4a404e9f2a4b40be292f0ad8d93807 2024-11-11T22:18:31,937 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/C of 20d39f0a33cd5786c48995acee2197ce into 4b4a404e9f2a4b40be292f0ad8d93807(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:31,937 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:31,937 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/C, priority=12, startTime=1731363511787; duration=0sec 2024-11-11T22:18:31,937 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:31,937 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:C 2024-11-11T22:18:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:32,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:18:32,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:32,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:32,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:32,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:32,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:32,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:32,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/8efcca8b1f2c40219a43bd9770c115cd is 50, key is test_row_0/A:col10/1731363512094/Put/seqid=0 2024-11-11T22:18:32,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742281_1457 (size=12001) 2024-11-11T22:18:32,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/8efcca8b1f2c40219a43bd9770c115cd 2024-11-11T22:18:32,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363572114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363572124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363572126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363572129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363572129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ee4682f49ab241b4a19734433832ae54 is 50, key is test_row_0/B:col10/1731363512094/Put/seqid=0 2024-11-11T22:18:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742282_1458 (size=12001) 2024-11-11T22:18:32,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363572240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363572240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363572249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363572249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363572249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363572449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363572449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363572459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363572459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363572459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ee4682f49ab241b4a19734433832ae54 2024-11-11T22:18:32,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/70e2e868dfa748be83f409127dcbdf37 is 50, key is test_row_0/C:col10/1731363512094/Put/seqid=0 2024-11-11T22:18:32,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742283_1459 (size=12001) 2024-11-11T22:18:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T22:18:32,580 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:32,580 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T22:18:32,582 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:32,584 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:32,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:32,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/70e2e868dfa748be83f409127dcbdf37 2024-11-11T22:18:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:18:32,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/8efcca8b1f2c40219a43bd9770c115cd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/8efcca8b1f2c40219a43bd9770c115cd 2024-11-11T22:18:32,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/8efcca8b1f2c40219a43bd9770c115cd, entries=150, sequenceid=106, filesize=11.7 K 2024-11-11T22:18:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ee4682f49ab241b4a19734433832ae54 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ee4682f49ab241b4a19734433832ae54 2024-11-11T22:18:32,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ee4682f49ab241b4a19734433832ae54, entries=150, sequenceid=106, filesize=11.7 K 2024-11-11T22:18:32,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/70e2e868dfa748be83f409127dcbdf37 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/70e2e868dfa748be83f409127dcbdf37 2024-11-11T22:18:32,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/70e2e868dfa748be83f409127dcbdf37, entries=150, sequenceid=106, filesize=11.7 K 2024-11-11T22:18:32,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 20d39f0a33cd5786c48995acee2197ce in 509ms, sequenceid=106, compaction requested=false 2024-11-11T22:18:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:18:32,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T22:18:32,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:32,740 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T22:18:32,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:32,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:32,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:32,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:32,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:32,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/4145711f10a04f1189a5ea09b6b7d121 is 50, key is test_row_0/A:col10/1731363512127/Put/seqid=0 2024-11-11T22:18:32,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742284_1460 (size=12001) 2024-11-11T22:18:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:32,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:32,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363572785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363572789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363572790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363572792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363572793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:18:32,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363572900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363572909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363572910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363572910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:32,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363572919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363573119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363573129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363573129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363573129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363573129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,155 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/4145711f10a04f1189a5ea09b6b7d121 2024-11-11T22:18:33,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/62c39caaffa4416185010b628241dbe5 is 50, key is test_row_0/B:col10/1731363512127/Put/seqid=0 2024-11-11T22:18:33,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742285_1461 (size=12001) 2024-11-11T22:18:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:18:33,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363573439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363573440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363573440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363573440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363573449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,573 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/62c39caaffa4416185010b628241dbe5 2024-11-11T22:18:33,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/cf92d8711bd64d79bbc8c096b54137ad is 50, key is test_row_0/C:col10/1731363512127/Put/seqid=0 2024-11-11T22:18:33,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742286_1462 (size=12001) 2024-11-11T22:18:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:18:33,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363573959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363573959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363573959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363573959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363573959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:33,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:33,985 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/cf92d8711bd64d79bbc8c096b54137ad 2024-11-11T22:18:33,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/4145711f10a04f1189a5ea09b6b7d121 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4145711f10a04f1189a5ea09b6b7d121 2024-11-11T22:18:33,993 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4145711f10a04f1189a5ea09b6b7d121, entries=150, sequenceid=121, filesize=11.7 K 2024-11-11T22:18:33,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/62c39caaffa4416185010b628241dbe5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/62c39caaffa4416185010b628241dbe5 2024-11-11T22:18:33,998 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/62c39caaffa4416185010b628241dbe5, entries=150, sequenceid=121, filesize=11.7 K 2024-11-11T22:18:33,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/cf92d8711bd64d79bbc8c096b54137ad as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/cf92d8711bd64d79bbc8c096b54137ad 2024-11-11T22:18:34,005 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/cf92d8711bd64d79bbc8c096b54137ad, entries=150, sequenceid=121, filesize=11.7 K 2024-11-11T22:18:34,006 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 20d39f0a33cd5786c48995acee2197ce in 1266ms, sequenceid=121, compaction requested=true 2024-11-11T22:18:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T22:18:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T22:18:34,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T22:18:34,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-11T22:18:34,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4300 sec 2024-11-11T22:18:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T22:18:34,729 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:34,730 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T22:18:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:18:34,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:34,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:34,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:18:34,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T22:18:34,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:34,888 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:18:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:34,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/587ce51d65344530990e0e8e40dae58f is 50, key is test_row_0/A:col10/1731363512783/Put/seqid=0 2024-11-11T22:18:34,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742287_1463 (size=12151) 2024-11-11T22:18:34,898 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/587ce51d65344530990e0e8e40dae58f 2024-11-11T22:18:34,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/7665d78e30854970af8166a98a3d4741 is 50, key is test_row_0/B:col10/1731363512783/Put/seqid=0 2024-11-11T22:18:34,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742288_1464 (size=12151) 2024-11-11T22:18:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:34,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:35,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363575002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363575004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363575006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363575010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363575015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:18:35,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363575119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363575119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363575129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363575129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363575129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,310 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/7665d78e30854970af8166a98a3d4741 2024-11-11T22:18:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/fa029df98ff842b48abf4bb4949ce166 is 50, key is test_row_0/C:col10/1731363512783/Put/seqid=0 2024-11-11T22:18:35,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742289_1465 (size=12151) 2024-11-11T22:18:35,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363575329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363575329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363575349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363575349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363575349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:18:35,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363575650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363575651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363575669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363575669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363575669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:35,721 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/fa029df98ff842b48abf4bb4949ce166 2024-11-11T22:18:35,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/587ce51d65344530990e0e8e40dae58f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/587ce51d65344530990e0e8e40dae58f 2024-11-11T22:18:35,729 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/587ce51d65344530990e0e8e40dae58f, entries=150, sequenceid=138, filesize=11.9 K 2024-11-11T22:18:35,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/7665d78e30854970af8166a98a3d4741 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7665d78e30854970af8166a98a3d4741 2024-11-11T22:18:35,733 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7665d78e30854970af8166a98a3d4741, entries=150, sequenceid=138, filesize=11.9 K 2024-11-11T22:18:35,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/fa029df98ff842b48abf4bb4949ce166 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/fa029df98ff842b48abf4bb4949ce166 2024-11-11T22:18:35,737 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/fa029df98ff842b48abf4bb4949ce166, entries=150, sequenceid=138, filesize=11.9 K 2024-11-11T22:18:35,738 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 20d39f0a33cd5786c48995acee2197ce in 850ms, sequenceid=138, compaction requested=true 2024-11-11T22:18:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:35,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T22:18:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T22:18:35,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T22:18:35,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-11T22:18:35,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0150 sec 2024-11-11T22:18:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T22:18:35,870 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:35,870 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-11T22:18:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:18:35,872 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:35,873 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:35,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:18:36,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-11T22:18:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:36,025 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T22:18:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:36,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:36,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/4191ba53d21140c493dd7db0f913725b is 50, key is test_row_0/A:col10/1731363514998/Put/seqid=0 2024-11-11T22:18:36,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742290_1466 (size=12151) 2024-11-11T22:18:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:36,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:18:36,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363576208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363576222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363576224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363576224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363576224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363576329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363576329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363576339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363576339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363576339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,443 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/4191ba53d21140c493dd7db0f913725b 2024-11-11T22:18:36,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/74786d2fdbac4a0e9205171ed6611507 is 50, key is test_row_0/B:col10/1731363514998/Put/seqid=0 2024-11-11T22:18:36,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742291_1467 (size=12151) 2024-11-11T22:18:36,468 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/74786d2fdbac4a0e9205171ed6611507 2024-11-11T22:18:36,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/19b8faf5e621431aad51e369e617f615 is 50, key is test_row_0/C:col10/1731363514998/Put/seqid=0 2024-11-11T22:18:36,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742292_1468 (size=12151) 2024-11-11T22:18:36,487 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/19b8faf5e621431aad51e369e617f615 2024-11-11T22:18:36,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/4191ba53d21140c493dd7db0f913725b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4191ba53d21140c493dd7db0f913725b 2024-11-11T22:18:36,496 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4191ba53d21140c493dd7db0f913725b, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T22:18:36,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/74786d2fdbac4a0e9205171ed6611507 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74786d2fdbac4a0e9205171ed6611507 2024-11-11T22:18:36,499 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74786d2fdbac4a0e9205171ed6611507, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T22:18:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:18:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/19b8faf5e621431aad51e369e617f615 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19b8faf5e621431aad51e369e617f615 2024-11-11T22:18:36,503 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19b8faf5e621431aad51e369e617f615, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T22:18:36,504 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=107.34 KB/109920 for 20d39f0a33cd5786c48995acee2197ce in 479ms, sequenceid=153, compaction requested=true 2024-11-11T22:18:36,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:36,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:36,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T22:18:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-11T22:18:36,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T22:18:36,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-11-11T22:18:36,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 639 msec 2024-11-11T22:18:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:36,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:18:36,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:36,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:36,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:36,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:36,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:36,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:36,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1a941d7e82814f89a3ac4b0ff1fe6f88 is 50, key is test_row_0/A:col10/1731363516544/Put/seqid=0 2024-11-11T22:18:36,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742293_1469 (size=14541) 2024-11-11T22:18:36,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363576568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363576569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363576570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363576571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363576576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363576689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363576689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363576689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363576689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363576689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363576910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363576910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363576910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363576910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363576910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:36,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1a941d7e82814f89a3ac4b0ff1fe6f88 2024-11-11T22:18:36,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ac69bcff46424846bb40248e2312af1d is 50, key is test_row_0/B:col10/1731363516544/Put/seqid=0 2024-11-11T22:18:36,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742294_1470 (size=12151) 2024-11-11T22:18:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T22:18:37,010 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:37,010 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:37,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T22:18:37,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:18:37,011 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:37,012 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:37,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:18:37,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:18:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363577219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363577219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363577219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363577229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363577229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:18:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:18:37,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ac69bcff46424846bb40248e2312af1d 2024-11-11T22:18:37,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/e14fdea8fcc946f9b7076b3ec73af30a is 50, key is test_row_0/C:col10/1731363516544/Put/seqid=0 2024-11-11T22:18:37,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742295_1471 (size=12151) 2024-11-11T22:18:37,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:18:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:18:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:37,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:18:37,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363577739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363577749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363577749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363577749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363577749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:37,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/e14fdea8fcc946f9b7076b3ec73af30a 2024-11-11T22:18:37,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:18:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:37,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:37,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1a941d7e82814f89a3ac4b0ff1fe6f88 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1a941d7e82814f89a3ac4b0ff1fe6f88 2024-11-11T22:18:37,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1a941d7e82814f89a3ac4b0ff1fe6f88, entries=200, sequenceid=173, filesize=14.2 K 2024-11-11T22:18:37,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ac69bcff46424846bb40248e2312af1d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ac69bcff46424846bb40248e2312af1d 2024-11-11T22:18:37,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ac69bcff46424846bb40248e2312af1d, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T22:18:37,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/e14fdea8fcc946f9b7076b3ec73af30a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e14fdea8fcc946f9b7076b3ec73af30a 2024-11-11T22:18:37,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e14fdea8fcc946f9b7076b3ec73af30a, entries=150, sequenceid=173, filesize=11.9 K 2024-11-11T22:18:37,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 20d39f0a33cd5786c48995acee2197ce in 1259ms, sequenceid=173, compaction requested=true 2024-11-11T22:18:37,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:37,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:37,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:37,804 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T22:18:37,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:37,804 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T22:18:37,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:37,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:37,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:37,807 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72594 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T22:18:37,807 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74984 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T22:18:37,807 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/B is initiating minor compaction (all files) 2024-11-11T22:18:37,807 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/A is initiating minor compaction (all files) 2024-11-11T22:18:37,807 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/B in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,807 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/A in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,808 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dc585f97d6df4b12b73e42dff9850716, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ee4682f49ab241b4a19734433832ae54, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/62c39caaffa4416185010b628241dbe5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7665d78e30854970af8166a98a3d4741, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74786d2fdbac4a0e9205171ed6611507, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ac69bcff46424846bb40248e2312af1d] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=70.9 K 2024-11-11T22:18:37,808 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/3a25fa886b7248ab9b9068336f4cfeb7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/8efcca8b1f2c40219a43bd9770c115cd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4145711f10a04f1189a5ea09b6b7d121, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/587ce51d65344530990e0e8e40dae58f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4191ba53d21140c493dd7db0f913725b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1a941d7e82814f89a3ac4b0ff1fe6f88] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=73.2 K 2024-11-11T22:18:37,808 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting dc585f97d6df4b12b73e42dff9850716, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731363510841 2024-11-11T22:18:37,808 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a25fa886b7248ab9b9068336f4cfeb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731363510841 2024-11-11T22:18:37,808 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ee4682f49ab241b4a19734433832ae54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731363510908 2024-11-11T22:18:37,808 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8efcca8b1f2c40219a43bd9770c115cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731363510908 2024-11-11T22:18:37,809 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4145711f10a04f1189a5ea09b6b7d121, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731363512108 2024-11-11T22:18:37,809 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c39caaffa4416185010b628241dbe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731363512108 2024-11-11T22:18:37,809 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 587ce51d65344530990e0e8e40dae58f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731363512783 2024-11-11T22:18:37,809 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7665d78e30854970af8166a98a3d4741, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731363512783 2024-11-11T22:18:37,809 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4191ba53d21140c493dd7db0f913725b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731363514998 2024-11-11T22:18:37,809 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 74786d2fdbac4a0e9205171ed6611507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731363514998 2024-11-11T22:18:37,810 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a941d7e82814f89a3ac4b0ff1fe6f88, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363516221 2024-11-11T22:18:37,810 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ac69bcff46424846bb40248e2312af1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363516221 2024-11-11T22:18:37,820 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:37,820 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/06c469bc6bd9426fa5cc948b925cdb49 is 50, key is test_row_0/B:col10/1731363516544/Put/seqid=0 2024-11-11T22:18:37,823 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#A#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:37,823 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/cfc895638ad94114911e29696377e699 is 50, key is test_row_0/A:col10/1731363516544/Put/seqid=0 2024-11-11T22:18:37,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742296_1472 (size=12493) 2024-11-11T22:18:37,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742297_1473 (size=12493) 2024-11-11T22:18:37,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T22:18:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:37,945 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:18:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:37,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:37,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/2b847b2d38a04b21bcc32bbc48273802 is 50, key is test_row_0/A:col10/1731363516568/Put/seqid=0 2024-11-11T22:18:37,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742298_1474 (size=12151) 2024-11-11T22:18:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:18:38,238 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/06c469bc6bd9426fa5cc948b925cdb49 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/06c469bc6bd9426fa5cc948b925cdb49 2024-11-11T22:18:38,240 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/cfc895638ad94114911e29696377e699 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cfc895638ad94114911e29696377e699 2024-11-11T22:18:38,243 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/B of 20d39f0a33cd5786c48995acee2197ce into 06c469bc6bd9426fa5cc948b925cdb49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:38,243 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:38,243 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/B, priority=10, startTime=1731363517804; duration=0sec 2024-11-11T22:18:38,243 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:38,243 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:B 2024-11-11T22:18:38,243 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-11T22:18:38,244 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72594 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-11T22:18:38,244 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/C is initiating minor compaction (all files) 2024-11-11T22:18:38,244 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/C in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:38,244 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4b4a404e9f2a4b40be292f0ad8d93807, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/70e2e868dfa748be83f409127dcbdf37, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/cf92d8711bd64d79bbc8c096b54137ad, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/fa029df98ff842b48abf4bb4949ce166, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19b8faf5e621431aad51e369e617f615, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e14fdea8fcc946f9b7076b3ec73af30a] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=70.9 K 2024-11-11T22:18:38,245 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4a404e9f2a4b40be292f0ad8d93807, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731363510841 2024-11-11T22:18:38,245 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e2e868dfa748be83f409127dcbdf37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731363510908 2024-11-11T22:18:38,246 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting cf92d8711bd64d79bbc8c096b54137ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731363512108 2024-11-11T22:18:38,247 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/A of 20d39f0a33cd5786c48995acee2197ce into cfc895638ad94114911e29696377e699(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:38,247 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:38,247 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/A, priority=10, startTime=1731363517804; duration=0sec 2024-11-11T22:18:38,247 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:38,247 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:A 2024-11-11T22:18:38,247 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fa029df98ff842b48abf4bb4949ce166, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731363512783 2024-11-11T22:18:38,247 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b8faf5e621431aad51e369e617f615, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731363514998 2024-11-11T22:18:38,248 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e14fdea8fcc946f9b7076b3ec73af30a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363516221 2024-11-11T22:18:38,263 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#C#compaction#402 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:38,264 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/4ebbce5ad77d4bb6bd80d7291c206366 is 50, key is test_row_0/C:col10/1731363516544/Put/seqid=0 2024-11-11T22:18:38,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742299_1475 (size=12493) 2024-11-11T22:18:38,354 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/2b847b2d38a04b21bcc32bbc48273802 2024-11-11T22:18:38,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d917d0265c264b0fb74f027b91465929 is 50, key is test_row_0/B:col10/1731363516568/Put/seqid=0 2024-11-11T22:18:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742300_1476 (size=12151) 2024-11-11T22:18:38,679 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/4ebbce5ad77d4bb6bd80d7291c206366 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4ebbce5ad77d4bb6bd80d7291c206366 2024-11-11T22:18:38,685 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/C of 20d39f0a33cd5786c48995acee2197ce into 4ebbce5ad77d4bb6bd80d7291c206366(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:38,685 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:38,685 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/C, priority=10, startTime=1731363517805; duration=0sec 2024-11-11T22:18:38,685 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:38,685 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:C 2024-11-11T22:18:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:38,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:38,765 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d917d0265c264b0fb74f027b91465929 2024-11-11T22:18:38,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/19f82eae497546beb0bf57b34ec914fb is 50, key is test_row_0/C:col10/1731363516568/Put/seqid=0 2024-11-11T22:18:38,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742301_1477 (size=12151) 2024-11-11T22:18:38,775 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/19f82eae497546beb0bf57b34ec914fb 2024-11-11T22:18:38,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/2b847b2d38a04b21bcc32bbc48273802 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b847b2d38a04b21bcc32bbc48273802 2024-11-11T22:18:38,782 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b847b2d38a04b21bcc32bbc48273802, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T22:18:38,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d917d0265c264b0fb74f027b91465929 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d917d0265c264b0fb74f027b91465929 2024-11-11T22:18:38,785 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d917d0265c264b0fb74f027b91465929, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T22:18:38,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/19f82eae497546beb0bf57b34ec914fb as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19f82eae497546beb0bf57b34ec914fb 2024-11-11T22:18:38,790 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19f82eae497546beb0bf57b34ec914fb, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T22:18:38,790 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=100.63 KB/103050 for 20d39f0a33cd5786c48995acee2197ce in 845ms, sequenceid=185, compaction requested=false 2024-11-11T22:18:38,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:38,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:38,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T22:18:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T22:18:38,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T22:18:38,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7790 sec 2024-11-11T22:18:38,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7830 sec 2024-11-11T22:18:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:38,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:18:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:38,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/bee82233d6554388990baa98a66df987 is 50, key is test_row_0/A:col10/1731363518792/Put/seqid=0 2024-11-11T22:18:38,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742302_1478 (size=14541) 2024-11-11T22:18:38,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363578806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363578809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363578810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363578814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363578815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363578929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363578929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363578929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363578929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363578929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363579149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363579149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363579149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363579149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363579149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T22:18:39,160 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:39,160 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T22:18:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:18:39,162 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:39,163 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:39,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:39,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/bee82233d6554388990baa98a66df987 2024-11-11T22:18:39,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/8afb980412214a9cbc5389b2b4b0e722 is 50, key is test_row_0/B:col10/1731363518792/Put/seqid=0 2024-11-11T22:18:39,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742303_1479 (size=12151) 2024-11-11T22:18:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:18:39,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:18:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:18:39,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:39,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363579462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363579462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363579462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363579469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363579470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:18:39,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/8afb980412214a9cbc5389b2b4b0e722 2024-11-11T22:18:39,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/040131083a9f4acd8fe0895d8ee0e94d is 50, key is test_row_0/C:col10/1731363518792/Put/seqid=0 2024-11-11T22:18:39,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:18:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742304_1480 (size=12151) 2024-11-11T22:18:39,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:18:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:39,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:18:39,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:18:39,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:39,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:39,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363579979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363579979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363579979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363579990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363579990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:40,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/040131083a9f4acd8fe0895d8ee0e94d 2024-11-11T22:18:40,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/bee82233d6554388990baa98a66df987 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/bee82233d6554388990baa98a66df987 2024-11-11T22:18:40,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/bee82233d6554388990baa98a66df987, entries=200, sequenceid=207, filesize=14.2 K 2024-11-11T22:18:40,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/8afb980412214a9cbc5389b2b4b0e722 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8afb980412214a9cbc5389b2b4b0e722 2024-11-11T22:18:40,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8afb980412214a9cbc5389b2b4b0e722, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T22:18:40,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/040131083a9f4acd8fe0895d8ee0e94d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/040131083a9f4acd8fe0895d8ee0e94d 2024-11-11T22:18:40,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/040131083a9f4acd8fe0895d8ee0e94d, entries=150, sequenceid=207, filesize=11.9 K 2024-11-11T22:18:40,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 20d39f0a33cd5786c48995acee2197ce in 1265ms, sequenceid=207, compaction requested=true 2024-11-11T22:18:40,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:40,059 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:18:40,061 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:40,062 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:40,062 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:40,062 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/B is initiating minor compaction (all files) 2024-11-11T22:18:40,062 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/B in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:40,062 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/06c469bc6bd9426fa5cc948b925cdb49, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d917d0265c264b0fb74f027b91465929, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8afb980412214a9cbc5389b2b4b0e722] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=35.9 K 2024-11-11T22:18:40,062 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/A is initiating minor compaction (all files) 2024-11-11T22:18:40,062 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/A in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:40,062 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cfc895638ad94114911e29696377e699, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b847b2d38a04b21bcc32bbc48273802, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/bee82233d6554388990baa98a66df987] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=38.3 K 2024-11-11T22:18:40,062 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc895638ad94114911e29696377e699, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363516221 2024-11-11T22:18:40,063 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c469bc6bd9426fa5cc948b925cdb49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363516221 2024-11-11T22:18:40,063 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b847b2d38a04b21bcc32bbc48273802, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731363516568 2024-11-11T22:18:40,063 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d917d0265c264b0fb74f027b91465929, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731363516568 2024-11-11T22:18:40,063 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting bee82233d6554388990baa98a66df987, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731363518786 2024-11-11T22:18:40,063 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8afb980412214a9cbc5389b2b4b0e722, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731363518787 2024-11-11T22:18:40,078 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:40,078 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/dcc272b2e1914bc28073b3f8aa90b71b is 50, key is test_row_0/B:col10/1731363518792/Put/seqid=0 2024-11-11T22:18:40,079 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:40,079 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f8a15eb5c59c4a328590c8375a5fd617 is 50, key is test_row_0/A:col10/1731363518792/Put/seqid=0 2024-11-11T22:18:40,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T22:18:40,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:40,085 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:18:40,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:40,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:40,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:40,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:40,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:40,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:40,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742306_1482 (size=12595) 2024-11-11T22:18:40,123 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f8a15eb5c59c4a328590c8375a5fd617 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f8a15eb5c59c4a328590c8375a5fd617 2024-11-11T22:18:40,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/6071316b646b45a2a27060541dab4033 is 50, key is test_row_0/A:col10/1731363518813/Put/seqid=0 2024-11-11T22:18:40,132 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/A of 20d39f0a33cd5786c48995acee2197ce into f8a15eb5c59c4a328590c8375a5fd617(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:40,132 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:40,132 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/A, priority=13, startTime=1731363520059; duration=0sec 2024-11-11T22:18:40,132 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:40,132 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:A 2024-11-11T22:18:40,132 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:40,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742305_1481 (size=12595) 2024-11-11T22:18:40,137 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:40,137 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/C is initiating minor compaction (all files) 2024-11-11T22:18:40,137 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/C in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:40,137 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4ebbce5ad77d4bb6bd80d7291c206366, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19f82eae497546beb0bf57b34ec914fb, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/040131083a9f4acd8fe0895d8ee0e94d] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=35.9 K 2024-11-11T22:18:40,137 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ebbce5ad77d4bb6bd80d7291c206366, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731363516221 2024-11-11T22:18:40,137 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19f82eae497546beb0bf57b34ec914fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731363516568 2024-11-11T22:18:40,138 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040131083a9f4acd8fe0895d8ee0e94d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731363518787 2024-11-11T22:18:40,158 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:40,158 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/7e922d3c26d2457f92c6775993c4f2ef is 50, key is test_row_0/C:col10/1731363518792/Put/seqid=0 2024-11-11T22:18:40,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742307_1483 (size=12151) 2024-11-11T22:18:40,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742308_1484 (size=12595) 2024-11-11T22:18:40,175 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/7e922d3c26d2457f92c6775993c4f2ef as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/7e922d3c26d2457f92c6775993c4f2ef 2024-11-11T22:18:40,180 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/C of 20d39f0a33cd5786c48995acee2197ce into 7e922d3c26d2457f92c6775993c4f2ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:40,180 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:40,180 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/C, priority=13, startTime=1731363520060; duration=0sec 2024-11-11T22:18:40,180 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:40,180 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:C 2024-11-11T22:18:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:18:40,543 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/dcc272b2e1914bc28073b3f8aa90b71b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dcc272b2e1914bc28073b3f8aa90b71b 2024-11-11T22:18:40,548 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/B of 20d39f0a33cd5786c48995acee2197ce into dcc272b2e1914bc28073b3f8aa90b71b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:40,548 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:40,548 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/B, priority=13, startTime=1731363520060; duration=0sec 2024-11-11T22:18:40,548 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:40,548 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:B 2024-11-11T22:18:40,562 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/6071316b646b45a2a27060541dab4033 2024-11-11T22:18:40,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/2f76f5c4646a4c6ebd610f1cf828c8fa is 50, key is test_row_0/B:col10/1731363518813/Put/seqid=0 2024-11-11T22:18:40,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742309_1485 (size=12151) 2024-11-11T22:18:40,975 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/2f76f5c4646a4c6ebd610f1cf828c8fa 2024-11-11T22:18:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/821e7395513f4812bd92032f3bf149fd is 50, key is test_row_0/C:col10/1731363518813/Put/seqid=0 2024-11-11T22:18:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:40,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:40,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742310_1486 (size=12151) 2024-11-11T22:18:41,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363581032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363581040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363581042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363581042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363581043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363581159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363581159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363581159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363581159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363581159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:18:41,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363581369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363581379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363581379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363581379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363581380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,396 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/821e7395513f4812bd92032f3bf149fd 2024-11-11T22:18:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/6071316b646b45a2a27060541dab4033 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/6071316b646b45a2a27060541dab4033 2024-11-11T22:18:41,402 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/6071316b646b45a2a27060541dab4033, entries=150, sequenceid=220, filesize=11.9 K 2024-11-11T22:18:41,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/2f76f5c4646a4c6ebd610f1cf828c8fa as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/2f76f5c4646a4c6ebd610f1cf828c8fa 2024-11-11T22:18:41,405 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/2f76f5c4646a4c6ebd610f1cf828c8fa, entries=150, sequenceid=220, filesize=11.9 K 2024-11-11T22:18:41,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/821e7395513f4812bd92032f3bf149fd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/821e7395513f4812bd92032f3bf149fd 2024-11-11T22:18:41,409 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/821e7395513f4812bd92032f3bf149fd, entries=150, sequenceid=220, filesize=11.9 K 2024-11-11T22:18:41,410 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 20d39f0a33cd5786c48995acee2197ce in 1325ms, sequenceid=220, compaction requested=false 2024-11-11T22:18:41,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:41,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:41,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T22:18:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T22:18:41,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T22:18:41,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2480 sec 2024-11-11T22:18:41,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2530 sec 2024-11-11T22:18:41,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:18:41,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:41,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:41,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:41,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:41,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:41,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:41,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/c01e0adb0135444285b0f35a38a68aa1 is 50, key is test_row_0/A:col10/1731363521023/Put/seqid=0 2024-11-11T22:18:41,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742311_1487 (size=14541) 2024-11-11T22:18:41,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363581708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363581709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363581710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363581711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363581711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363581819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363581820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363581829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363581829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363581829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363582040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363582040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363582040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363582040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363582040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/c01e0adb0135444285b0f35a38a68aa1 2024-11-11T22:18:42,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d5da66c367a1407da4ef116412181fbd is 50, key is test_row_0/B:col10/1731363521023/Put/seqid=0 2024-11-11T22:18:42,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742312_1488 (size=12151) 2024-11-11T22:18:42,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363582349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363582359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363582360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363582360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363582360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d5da66c367a1407da4ef116412181fbd 2024-11-11T22:18:42,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/17922ef889074c7287d1ef1e39ad6ad1 is 50, key is test_row_0/C:col10/1731363521023/Put/seqid=0 2024-11-11T22:18:42,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742313_1489 (size=12151) 2024-11-11T22:18:42,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363582870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363582880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363582880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363582880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363582880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:42,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:42,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/17922ef889074c7287d1ef1e39ad6ad1 2024-11-11T22:18:42,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/c01e0adb0135444285b0f35a38a68aa1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c01e0adb0135444285b0f35a38a68aa1 2024-11-11T22:18:42,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c01e0adb0135444285b0f35a38a68aa1, entries=200, sequenceid=243, filesize=14.2 K 2024-11-11T22:18:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d5da66c367a1407da4ef116412181fbd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d5da66c367a1407da4ef116412181fbd 2024-11-11T22:18:42,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d5da66c367a1407da4ef116412181fbd, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T22:18:42,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/17922ef889074c7287d1ef1e39ad6ad1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/17922ef889074c7287d1ef1e39ad6ad1 2024-11-11T22:18:42,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/17922ef889074c7287d1ef1e39ad6ad1, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T22:18:42,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 20d39f0a33cd5786c48995acee2197ce in 1262ms, sequenceid=243, compaction requested=true 2024-11-11T22:18:42,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:42,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:42,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:42,956 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:42,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:42,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:42,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:42,956 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:42,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:42,957 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:42,957 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:42,957 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/B is initiating minor compaction (all files) 2024-11-11T22:18:42,957 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/A is initiating minor compaction (all files) 2024-11-11T22:18:42,957 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/B in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:42,957 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/A in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:42,957 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f8a15eb5c59c4a328590c8375a5fd617, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/6071316b646b45a2a27060541dab4033, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c01e0adb0135444285b0f35a38a68aa1] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=38.4 K 2024-11-11T22:18:42,957 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dcc272b2e1914bc28073b3f8aa90b71b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/2f76f5c4646a4c6ebd610f1cf828c8fa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d5da66c367a1407da4ef116412181fbd] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.0 K 2024-11-11T22:18:42,959 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8a15eb5c59c4a328590c8375a5fd617, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731363518787 2024-11-11T22:18:42,959 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc272b2e1914bc28073b3f8aa90b71b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731363518787 2024-11-11T22:18:42,959 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f76f5c4646a4c6ebd610f1cf828c8fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363518807 2024-11-11T22:18:42,959 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6071316b646b45a2a27060541dab4033, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363518807 2024-11-11T22:18:42,959 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d5da66c367a1407da4ef116412181fbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731363521023 2024-11-11T22:18:42,959 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c01e0adb0135444285b0f35a38a68aa1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731363521023 2024-11-11T22:18:42,964 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#B#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:42,964 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:42,965 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/76d298e1f7494bddb476314b793b12de is 50, key is test_row_0/B:col10/1731363521023/Put/seqid=0 2024-11-11T22:18:42,965 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/55e97dd8512647beb0d2e5072fdd3c2f is 50, key is test_row_0/A:col10/1731363521023/Put/seqid=0 2024-11-11T22:18:42,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742314_1490 (size=12697) 2024-11-11T22:18:42,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742315_1491 (size=12697) 2024-11-11T22:18:43,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T22:18:43,340 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:43,341 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T22:18:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:18:43,343 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:43,344 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:43,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:43,372 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/76d298e1f7494bddb476314b793b12de as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/76d298e1f7494bddb476314b793b12de 2024-11-11T22:18:43,374 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/55e97dd8512647beb0d2e5072fdd3c2f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/55e97dd8512647beb0d2e5072fdd3c2f 2024-11-11T22:18:43,376 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/B of 20d39f0a33cd5786c48995acee2197ce into 76d298e1f7494bddb476314b793b12de(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:43,376 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:43,376 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/B, priority=13, startTime=1731363522956; duration=0sec 2024-11-11T22:18:43,376 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:43,376 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:B 2024-11-11T22:18:43,376 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:43,377 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:43,377 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/C is initiating minor compaction (all files) 2024-11-11T22:18:43,377 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/A of 20d39f0a33cd5786c48995acee2197ce into 55e97dd8512647beb0d2e5072fdd3c2f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:43,377 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:43,377 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/C in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:43,377 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/A, priority=13, startTime=1731363522956; duration=0sec 2024-11-11T22:18:43,378 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/7e922d3c26d2457f92c6775993c4f2ef, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/821e7395513f4812bd92032f3bf149fd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/17922ef889074c7287d1ef1e39ad6ad1] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.0 K 2024-11-11T22:18:43,378 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:43,378 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:A 2024-11-11T22:18:43,378 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e922d3c26d2457f92c6775993c4f2ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731363518787 2024-11-11T22:18:43,378 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 821e7395513f4812bd92032f3bf149fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731363518807 2024-11-11T22:18:43,378 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 17922ef889074c7287d1ef1e39ad6ad1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731363521023 2024-11-11T22:18:43,390 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:43,390 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/458a9575f43543d48887af94c6804aef is 50, key is test_row_0/C:col10/1731363521023/Put/seqid=0 2024-11-11T22:18:43,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742316_1492 (size=12697) 2024-11-11T22:18:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:18:43,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T22:18:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:43,502 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:18:43,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:43,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:43,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:43,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:43,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:43,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f205394874854fe2b7f0595b53940b67 is 50, key is test_row_0/A:col10/1731363521709/Put/seqid=0 2024-11-11T22:18:43,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742317_1493 (size=12151) 2024-11-11T22:18:43,532 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f205394874854fe2b7f0595b53940b67 2024-11-11T22:18:43,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/94f47229c55f4e94a1793cea28b2fb1a is 50, key is test_row_0/B:col10/1731363521709/Put/seqid=0 2024-11-11T22:18:43,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742318_1494 (size=12151) 2024-11-11T22:18:43,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:18:43,811 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/458a9575f43543d48887af94c6804aef as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/458a9575f43543d48887af94c6804aef 2024-11-11T22:18:43,816 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/C of 20d39f0a33cd5786c48995acee2197ce into 458a9575f43543d48887af94c6804aef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:43,816 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:43,816 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/C, priority=13, startTime=1731363522956; duration=0sec 2024-11-11T22:18:43,816 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:43,816 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:C 2024-11-11T22:18:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:43,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:43,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363583927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:43,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363583928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:43,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363583934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:43,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:43,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363523925, max timestamp=1731363523934Num uniques 859; 2024-11-11T22:18:43,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363523925, max timestamp=1731363523934Num uniques 859; 2024-11-11T22:18:43,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731363523925, max timestamp=1731363523934Num uniques 859; 2024-11-11T22:18:43,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:18:43,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:18:43,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T22:18:43,954 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/94f47229c55f4e94a1793cea28b2fb1a 2024-11-11T22:18:43,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/0fb83b14a1dc4c8484d329ed09b7fc87 is 50, key is test_row_0/C:col10/1731363521709/Put/seqid=0 2024-11-11T22:18:43,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742319_1495 (size=12151) 2024-11-11T22:18:43,964 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/0fb83b14a1dc4c8484d329ed09b7fc87 2024-11-11T22:18:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:18:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f205394874854fe2b7f0595b53940b67 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f205394874854fe2b7f0595b53940b67 2024-11-11T22:18:43,974 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f205394874854fe2b7f0595b53940b67, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T22:18:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/94f47229c55f4e94a1793cea28b2fb1a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94f47229c55f4e94a1793cea28b2fb1a 2024-11-11T22:18:43,984 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94f47229c55f4e94a1793cea28b2fb1a, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T22:18:43,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/0fb83b14a1dc4c8484d329ed09b7fc87 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0fb83b14a1dc4c8484d329ed09b7fc87 2024-11-11T22:18:43,993 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0fb83b14a1dc4c8484d329ed09b7fc87, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T22:18:43,994 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=80.51 KB/82440 for 20d39f0a33cd5786c48995acee2197ce in 493ms, sequenceid=257, compaction requested=false 2024-11-11T22:18:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:43,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T22:18:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T22:18:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:43,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=87.22 KB heapSize=245.53 KB 2024-11-11T22:18:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:43,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:44,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T22:18:44,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 651 msec 2024-11-11T22:18:44,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/ff3f33de91504cf0aa7e330a9529d9d4 is 50, key is test_row_0/A:col10/1731363523994/Put/seqid=0 2024-11-11T22:18:44,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 660 msec 2024-11-11T22:18:44,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742320_1496 (size=14741) 2024-11-11T22:18:44,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/ff3f33de91504cf0aa7e330a9529d9d4 2024-11-11T22:18:44,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/b07fee7a37564c269e71505997870732 is 50, key is test_row_0/B:col10/1731363523994/Put/seqid=0 2024-11-11T22:18:44,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742321_1497 (size=12301) 2024-11-11T22:18:44,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363584053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363584052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363584054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363584090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363584090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363584199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363584199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363584200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363584219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363584219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363584410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363584410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363584411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/b07fee7a37564c269e71505997870732 2024-11-11T22:18:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363584430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363584430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/1912f51d1c17417f9570b5d887ee75ea is 50, key is test_row_0/C:col10/1731363523994/Put/seqid=0 2024-11-11T22:18:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T22:18:44,479 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:44,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-11T22:18:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T22:18:44,484 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:44,486 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:44,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:44,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742322_1498 (size=12301) 2024-11-11T22:18:44,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/1912f51d1c17417f9570b5d887ee75ea 2024-11-11T22:18:44,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/ff3f33de91504cf0aa7e330a9529d9d4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/ff3f33de91504cf0aa7e330a9529d9d4 2024-11-11T22:18:44,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/ff3f33de91504cf0aa7e330a9529d9d4, entries=200, sequenceid=288, filesize=14.4 K 2024-11-11T22:18:44,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/b07fee7a37564c269e71505997870732 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/b07fee7a37564c269e71505997870732 2024-11-11T22:18:44,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/b07fee7a37564c269e71505997870732, entries=150, sequenceid=288, filesize=12.0 K 2024-11-11T22:18:44,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/1912f51d1c17417f9570b5d887ee75ea as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1912f51d1c17417f9570b5d887ee75ea 2024-11-11T22:18:44,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1912f51d1c17417f9570b5d887ee75ea, entries=150, sequenceid=288, filesize=12.0 K 2024-11-11T22:18:44,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~245.48 KB/251376, currentSize=100.63 KB/103050 for 20d39f0a33cd5786c48995acee2197ce in 537ms, sequenceid=288, compaction requested=true 2024-11-11T22:18:44,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:44,534 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:44,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:44,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:44,535 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:44,536 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:44,536 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/A is initiating minor compaction (all files) 2024-11-11T22:18:44,536 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/A in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:44,536 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/55e97dd8512647beb0d2e5072fdd3c2f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f205394874854fe2b7f0595b53940b67, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/ff3f33de91504cf0aa7e330a9529d9d4] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=38.7 K 2024-11-11T22:18:44,538 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:44,538 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/B is initiating minor compaction (all files) 2024-11-11T22:18:44,538 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/B in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:44,538 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/76d298e1f7494bddb476314b793b12de, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94f47229c55f4e94a1793cea28b2fb1a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/b07fee7a37564c269e71505997870732] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.3 K 2024-11-11T22:18:44,539 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e97dd8512647beb0d2e5072fdd3c2f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731363521023 2024-11-11T22:18:44,540 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f205394874854fe2b7f0595b53940b67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731363521707 2024-11-11T22:18:44,540 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d298e1f7494bddb476314b793b12de, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731363521023 2024-11-11T22:18:44,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:44,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:44,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:44,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:44,540 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff3f33de91504cf0aa7e330a9529d9d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363523966 2024-11-11T22:18:44,540 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 94f47229c55f4e94a1793cea28b2fb1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731363521707 2024-11-11T22:18:44,541 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b07fee7a37564c269e71505997870732, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363523977 2024-11-11T22:18:44,558 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:44,559 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/eb071683743a4494952b8f99262121e5 is 50, key is test_row_0/B:col10/1731363523994/Put/seqid=0 2024-11-11T22:18:44,567 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#A#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:44,567 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/324b4738d7a2435ea320032a3dd37c33 is 50, key is test_row_0/A:col10/1731363523994/Put/seqid=0 2024-11-11T22:18:44,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742323_1499 (size=12949) 2024-11-11T22:18:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T22:18:44,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742324_1500 (size=12949) 2024-11-11T22:18:44,614 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/324b4738d7a2435ea320032a3dd37c33 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/324b4738d7a2435ea320032a3dd37c33 2024-11-11T22:18:44,627 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/A of 20d39f0a33cd5786c48995acee2197ce into 324b4738d7a2435ea320032a3dd37c33(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:44,627 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:44,628 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/A, priority=13, startTime=1731363524534; duration=0sec 2024-11-11T22:18:44,628 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:44,628 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:A 2024-11-11T22:18:44,628 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:44,629 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:44,629 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/C is initiating minor compaction (all files) 2024-11-11T22:18:44,629 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/C in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:44,629 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/458a9575f43543d48887af94c6804aef, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0fb83b14a1dc4c8484d329ed09b7fc87, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1912f51d1c17417f9570b5d887ee75ea] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.3 K 2024-11-11T22:18:44,629 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 458a9575f43543d48887af94c6804aef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731363521023 2024-11-11T22:18:44,629 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fb83b14a1dc4c8484d329ed09b7fc87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731363521707 2024-11-11T22:18:44,630 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1912f51d1c17417f9570b5d887ee75ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363523977 2024-11-11T22:18:44,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T22:18:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:44,644 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T22:18:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:44,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:44,651 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#C#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:44,651 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/e6c5c92da52d4e8cb3725889a6d09e7a is 50, key is test_row_0/C:col10/1731363523994/Put/seqid=0 2024-11-11T22:18:44,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/cc1194d1256a412b837adaae054e7a63 is 50, key is test_row_0/A:col10/1731363524053/Put/seqid=0 2024-11-11T22:18:44,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742325_1501 (size=12949) 2024-11-11T22:18:44,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742326_1502 (size=12301) 2024-11-11T22:18:44,694 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/cc1194d1256a412b837adaae054e7a63 2024-11-11T22:18:44,697 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/e6c5c92da52d4e8cb3725889a6d09e7a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e6c5c92da52d4e8cb3725889a6d09e7a 2024-11-11T22:18:44,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/a625ae7e021c42c2ba5cf25b5bec4791 is 50, key is test_row_0/B:col10/1731363524053/Put/seqid=0 2024-11-11T22:18:44,710 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/C of 20d39f0a33cd5786c48995acee2197ce into e6c5c92da52d4e8cb3725889a6d09e7a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:44,710 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:44,710 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/C, priority=13, startTime=1731363524540; duration=0sec 2024-11-11T22:18:44,711 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:44,711 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:C 2024-11-11T22:18:44,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742327_1503 (size=12301) 2024-11-11T22:18:44,726 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/a625ae7e021c42c2ba5cf25b5bec4791 2024-11-11T22:18:44,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/e83c4370ba7c477e8a4fab9765e6110d is 50, key is test_row_0/C:col10/1731363524053/Put/seqid=0 2024-11-11T22:18:44,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:44,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742328_1504 (size=12301) 2024-11-11T22:18:44,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T22:18:44,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363584768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363584776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363584817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363584817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363584817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363584919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363584920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363584939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363584939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363584939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:44,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:44,990 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/eb071683743a4494952b8f99262121e5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/eb071683743a4494952b8f99262121e5 2024-11-11T22:18:44,994 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/B of 20d39f0a33cd5786c48995acee2197ce into eb071683743a4494952b8f99262121e5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:44,994 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:44,994 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/B, priority=13, startTime=1731363524535; duration=0sec 2024-11-11T22:18:44,994 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:44,994 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:B 2024-11-11T22:18:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T22:18:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363585129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363585129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363585159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363585159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363585160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,176 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/e83c4370ba7c477e8a4fab9765e6110d 2024-11-11T22:18:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/cc1194d1256a412b837adaae054e7a63 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cc1194d1256a412b837adaae054e7a63 2024-11-11T22:18:45,184 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cc1194d1256a412b837adaae054e7a63, entries=150, sequenceid=307, filesize=12.0 K 2024-11-11T22:18:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/a625ae7e021c42c2ba5cf25b5bec4791 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a625ae7e021c42c2ba5cf25b5bec4791 2024-11-11T22:18:45,190 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a625ae7e021c42c2ba5cf25b5bec4791, entries=150, sequenceid=307, filesize=12.0 K 2024-11-11T22:18:45,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/e83c4370ba7c477e8a4fab9765e6110d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e83c4370ba7c477e8a4fab9765e6110d 2024-11-11T22:18:45,195 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e83c4370ba7c477e8a4fab9765e6110d, entries=150, sequenceid=307, filesize=12.0 K 2024-11-11T22:18:45,196 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 20d39f0a33cd5786c48995acee2197ce in 553ms, sequenceid=307, compaction requested=false 2024-11-11T22:18:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-11T22:18:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-11T22:18:45,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T22:18:45,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 711 msec 2024-11-11T22:18:45,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 719 msec 2024-11-11T22:18:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:45,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T22:18:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:45,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:45,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1f8b1736ebab448da07b4aacd88d8d64 is 50, key is test_row_0/A:col10/1731363525442/Put/seqid=0 2024-11-11T22:18:45,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742329_1505 (size=14741) 2024-11-11T22:18:45,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363585478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363585480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363585481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363585482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363585488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363585599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363585599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363585609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363585609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363585609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T22:18:45,619 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:45,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-11T22:18:45,621 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T22:18:45,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:45,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T22:18:45,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T22:18:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:45,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:45,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363585820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363585821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363585829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363585829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363585829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:45,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:45,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1f8b1736ebab448da07b4aacd88d8d64 2024-11-11T22:18:45,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/8cdf4c60e0134a79ba7bb083ec3cba2f is 50, key is test_row_0/B:col10/1731363525442/Put/seqid=0 2024-11-11T22:18:45,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742330_1506 (size=12301) 2024-11-11T22:18:45,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/8cdf4c60e0134a79ba7bb083ec3cba2f 2024-11-11T22:18:45,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/d910bcba203c454d96b325e65cb0d3f9 is 50, key is test_row_0/C:col10/1731363525442/Put/seqid=0 2024-11-11T22:18:45,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742331_1507 (size=12301) 2024-11-11T22:18:45,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/d910bcba203c454d96b325e65cb0d3f9 2024-11-11T22:18:45,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/1f8b1736ebab448da07b4aacd88d8d64 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1f8b1736ebab448da07b4aacd88d8d64 2024-11-11T22:18:45,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1f8b1736ebab448da07b4aacd88d8d64, entries=200, sequenceid=324, filesize=14.4 K 2024-11-11T22:18:45,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/8cdf4c60e0134a79ba7bb083ec3cba2f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8cdf4c60e0134a79ba7bb083ec3cba2f 2024-11-11T22:18:45,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8cdf4c60e0134a79ba7bb083ec3cba2f, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T22:18:45,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/d910bcba203c454d96b325e65cb0d3f9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/d910bcba203c454d96b325e65cb0d3f9 2024-11-11T22:18:45,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/d910bcba203c454d96b325e65cb0d3f9, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T22:18:45,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 20d39f0a33cd5786c48995acee2197ce in 461ms, sequenceid=324, compaction requested=true 2024-11-11T22:18:45,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:45,904 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:45,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:45,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:45,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:45,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:45,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:45,905 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:45,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:45,906 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:45,906 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:45,906 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/B is initiating minor compaction (all files) 2024-11-11T22:18:45,906 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/A is initiating minor compaction (all files) 2024-11-11T22:18:45,906 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/A in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:45,906 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/B in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:45,906 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/eb071683743a4494952b8f99262121e5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a625ae7e021c42c2ba5cf25b5bec4791, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8cdf4c60e0134a79ba7bb083ec3cba2f] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.7 K 2024-11-11T22:18:45,906 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/324b4738d7a2435ea320032a3dd37c33, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cc1194d1256a412b837adaae054e7a63, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1f8b1736ebab448da07b4aacd88d8d64] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=39.1 K 2024-11-11T22:18:45,907 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 324b4738d7a2435ea320032a3dd37c33, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363523977 2024-11-11T22:18:45,907 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting eb071683743a4494952b8f99262121e5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363523977 2024-11-11T22:18:45,907 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a625ae7e021c42c2ba5cf25b5bec4791, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731363524053 2024-11-11T22:18:45,907 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1194d1256a412b837adaae054e7a63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731363524053 2024-11-11T22:18:45,907 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cdf4c60e0134a79ba7bb083ec3cba2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363524760 2024-11-11T22:18:45,907 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f8b1736ebab448da07b4aacd88d8d64, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363524760 2024-11-11T22:18:45,915 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:45,916 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/32aac6e935844b1d8380e40fd18cd7be is 50, key is test_row_0/A:col10/1731363525442/Put/seqid=0 2024-11-11T22:18:45,927 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:45,927 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/0d9f8f40ea0f4c3799532650c0a1514e is 50, key is test_row_0/B:col10/1731363525442/Put/seqid=0 2024-11-11T22:18:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T22:18:45,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T22:18:45,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:45,941 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:18:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:45,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742332_1508 (size=13051) 2024-11-11T22:18:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/a68580c3b4694445a418e30ff501e02f is 50, key is test_row_0/A:col10/1731363525486/Put/seqid=0 2024-11-11T22:18:45,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742333_1509 (size=13051) 2024-11-11T22:18:45,970 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/32aac6e935844b1d8380e40fd18cd7be as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/32aac6e935844b1d8380e40fd18cd7be 2024-11-11T22:18:45,977 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/A of 20d39f0a33cd5786c48995acee2197ce into 32aac6e935844b1d8380e40fd18cd7be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:45,977 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:45,977 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/A, priority=13, startTime=1731363525904; duration=0sec 2024-11-11T22:18:45,977 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:45,977 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:A 2024-11-11T22:18:45,977 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:45,978 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:45,978 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/C is initiating minor compaction (all files) 2024-11-11T22:18:45,978 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/C in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:45,978 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e6c5c92da52d4e8cb3725889a6d09e7a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e83c4370ba7c477e8a4fab9765e6110d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/d910bcba203c454d96b325e65cb0d3f9] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.7 K 2024-11-11T22:18:45,979 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6c5c92da52d4e8cb3725889a6d09e7a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731363523977 2024-11-11T22:18:45,979 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e83c4370ba7c477e8a4fab9765e6110d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731363524053 2024-11-11T22:18:45,979 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting d910bcba203c454d96b325e65cb0d3f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363524760 2024-11-11T22:18:45,985 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/0d9f8f40ea0f4c3799532650c0a1514e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/0d9f8f40ea0f4c3799532650c0a1514e 2024-11-11T22:18:45,990 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/B of 20d39f0a33cd5786c48995acee2197ce into 0d9f8f40ea0f4c3799532650c0a1514e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:45,990 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:45,990 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/B, priority=13, startTime=1731363525905; duration=0sec 2024-11-11T22:18:45,990 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:45,990 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:B 2024-11-11T22:18:45,993 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#C#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:45,993 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/0e8eab0030d04e4ca63fa44c87d5057e is 50, key is test_row_0/C:col10/1731363525442/Put/seqid=0 2024-11-11T22:18:46,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742334_1510 (size=12301) 2024-11-11T22:18:46,007 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/a68580c3b4694445a418e30ff501e02f 2024-11-11T22:18:46,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742335_1511 (size=13051) 2024-11-11T22:18:46,014 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/0e8eab0030d04e4ca63fa44c87d5057e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0e8eab0030d04e4ca63fa44c87d5057e 2024-11-11T22:18:46,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/a8407518b194492b96808c73444638ba is 50, key is test_row_0/B:col10/1731363525486/Put/seqid=0 2024-11-11T22:18:46,029 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/C of 20d39f0a33cd5786c48995acee2197ce into 0e8eab0030d04e4ca63fa44c87d5057e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:46,029 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:46,029 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/C, priority=13, startTime=1731363525905; duration=0sec 2024-11-11T22:18:46,029 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:46,029 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:C 2024-11-11T22:18:46,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742336_1512 (size=12301) 2024-11-11T22:18:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:46,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:46,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363586161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363586163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363586164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363586174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363586175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T22:18:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363586280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363586280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363586280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363586280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363586289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,445 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/a8407518b194492b96808c73444638ba 2024-11-11T22:18:46,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/00d6e82b3fdd4bba928834734a7d1064 is 50, key is test_row_0/C:col10/1731363525486/Put/seqid=0 2024-11-11T22:18:46,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742337_1513 (size=12301) 2024-11-11T22:18:46,454 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/00d6e82b3fdd4bba928834734a7d1064 2024-11-11T22:18:46,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/a68580c3b4694445a418e30ff501e02f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/a68580c3b4694445a418e30ff501e02f 2024-11-11T22:18:46,462 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/a68580c3b4694445a418e30ff501e02f, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T22:18:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/a8407518b194492b96808c73444638ba as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a8407518b194492b96808c73444638ba 2024-11-11T22:18:46,468 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a8407518b194492b96808c73444638ba, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T22:18:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/00d6e82b3fdd4bba928834734a7d1064 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/00d6e82b3fdd4bba928834734a7d1064 2024-11-11T22:18:46,474 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/00d6e82b3fdd4bba928834734a7d1064, entries=150, sequenceid=341, filesize=12.0 K 2024-11-11T22:18:46,475 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 20d39f0a33cd5786c48995acee2197ce in 534ms, sequenceid=341, compaction requested=false 2024-11-11T22:18:46,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:46,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:46,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-11T22:18:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-11T22:18:46,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T22:18:46,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 853 msec 2024-11-11T22:18:46,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 863 msec 2024-11-11T22:18:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:46,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T22:18:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:46,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:46,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/71fb5cb271fc4e0fb4fee03c2fdf150b is 50, key is test_row_0/A:col10/1731363526172/Put/seqid=0 2024-11-11T22:18:46,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742338_1514 (size=14741) 2024-11-11T22:18:46,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/71fb5cb271fc4e0fb4fee03c2fdf150b 2024-11-11T22:18:46,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d6e9b4ccf5484a21aa2d3498055d43ab is 50, key is test_row_0/B:col10/1731363526172/Put/seqid=0 2024-11-11T22:18:46,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363586512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363586514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742339_1515 (size=12301) 2024-11-11T22:18:46,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363586535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363586540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363586540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363586640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363586649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363586659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363586659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363586659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T22:18:46,759 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:46,760 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-11T22:18:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T22:18:46,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:46,762 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:46,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:46,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363586850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363586859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T22:18:46,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363586879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363586879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363586879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:46,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T22:18:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:46,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:46,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:46,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d6e9b4ccf5484a21aa2d3498055d43ab 2024-11-11T22:18:46,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/bb6eeed4af5a4395bdb2ef8767770c7d is 50, key is test_row_0/C:col10/1731363526172/Put/seqid=0 2024-11-11T22:18:46,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742340_1516 (size=12301) 2024-11-11T22:18:47,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T22:18:47,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:47,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:47,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T22:18:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363587169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363587179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363587199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363587199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,206 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363587200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T22:18:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:47,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/bb6eeed4af5a4395bdb2ef8767770c7d 2024-11-11T22:18:47,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/71fb5cb271fc4e0fb4fee03c2fdf150b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/71fb5cb271fc4e0fb4fee03c2fdf150b 2024-11-11T22:18:47,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/71fb5cb271fc4e0fb4fee03c2fdf150b, entries=200, sequenceid=360, filesize=14.4 K 2024-11-11T22:18:47,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/d6e9b4ccf5484a21aa2d3498055d43ab as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d6e9b4ccf5484a21aa2d3498055d43ab 2024-11-11T22:18:47,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d6e9b4ccf5484a21aa2d3498055d43ab, entries=150, sequenceid=360, filesize=12.0 K 2024-11-11T22:18:47,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/bb6eeed4af5a4395bdb2ef8767770c7d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/bb6eeed4af5a4395bdb2ef8767770c7d 2024-11-11T22:18:47,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/bb6eeed4af5a4395bdb2ef8767770c7d, entries=150, sequenceid=360, filesize=12.0 K 2024-11-11T22:18:47,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 20d39f0a33cd5786c48995acee2197ce in 874ms, sequenceid=360, compaction requested=true 2024-11-11T22:18:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:47,369 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:47,369 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:47,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:47,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T22:18:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,372 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:18:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:47,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:47,372 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:47,372 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/A is initiating minor compaction (all files) 2024-11-11T22:18:47,372 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:47,373 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/B is initiating minor compaction (all files) 2024-11-11T22:18:47,373 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/A in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,373 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/B in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,373 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/32aac6e935844b1d8380e40fd18cd7be, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/a68580c3b4694445a418e30ff501e02f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/71fb5cb271fc4e0fb4fee03c2fdf150b] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=39.2 K 2024-11-11T22:18:47,373 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/0d9f8f40ea0f4c3799532650c0a1514e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a8407518b194492b96808c73444638ba, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d6e9b4ccf5484a21aa2d3498055d43ab] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.8 K 2024-11-11T22:18:47,373 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32aac6e935844b1d8380e40fd18cd7be, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363524760 2024-11-11T22:18:47,373 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d9f8f40ea0f4c3799532650c0a1514e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363524760 2024-11-11T22:18:47,373 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a68580c3b4694445a418e30ff501e02f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731363525478 2024-11-11T22:18:47,373 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a8407518b194492b96808c73444638ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731363525478 2024-11-11T22:18:47,373 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71fb5cb271fc4e0fb4fee03c2fdf150b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731363526162 2024-11-11T22:18:47,374 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e9b4ccf5484a21aa2d3498055d43ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731363526172 2024-11-11T22:18:47,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/578d7fabaf1141f6a17fba66adc2072e is 50, key is test_row_0/A:col10/1731363526527/Put/seqid=0 2024-11-11T22:18:47,386 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:47,387 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/c9eeb5684766420c94b97b791c0d2ad6 is 50, key is test_row_0/A:col10/1731363526172/Put/seqid=0 2024-11-11T22:18:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T22:18:47,393 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:47,394 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ea9b028355f4464285631557b44e0c63 is 50, key is test_row_0/B:col10/1731363526172/Put/seqid=0 2024-11-11T22:18:47,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742341_1517 (size=12301) 2024-11-11T22:18:47,395 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/578d7fabaf1141f6a17fba66adc2072e 2024-11-11T22:18:47,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ccc67416b16b4b1f8b4f59bd99199fce is 50, key is test_row_0/B:col10/1731363526527/Put/seqid=0 2024-11-11T22:18:47,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742342_1518 (size=13153) 2024-11-11T22:18:47,424 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/c9eeb5684766420c94b97b791c0d2ad6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c9eeb5684766420c94b97b791c0d2ad6 2024-11-11T22:18:47,427 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/A of 20d39f0a33cd5786c48995acee2197ce into c9eeb5684766420c94b97b791c0d2ad6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:47,428 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:47,428 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/A, priority=13, startTime=1731363527369; duration=0sec 2024-11-11T22:18:47,428 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:47,428 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:A 2024-11-11T22:18:47,428 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:47,429 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:47,429 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/C is initiating minor compaction (all files) 2024-11-11T22:18:47,429 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/C in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,429 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0e8eab0030d04e4ca63fa44c87d5057e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/00d6e82b3fdd4bba928834734a7d1064, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/bb6eeed4af5a4395bdb2ef8767770c7d] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.8 K 2024-11-11T22:18:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742343_1519 (size=13153) 2024-11-11T22:18:47,429 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e8eab0030d04e4ca63fa44c87d5057e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731363524760 2024-11-11T22:18:47,430 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00d6e82b3fdd4bba928834734a7d1064, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731363525478 2024-11-11T22:18:47,430 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb6eeed4af5a4395bdb2ef8767770c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731363526172 2024-11-11T22:18:47,436 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ea9b028355f4464285631557b44e0c63 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ea9b028355f4464285631557b44e0c63 2024-11-11T22:18:47,436 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:47,437 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/50191b5501a744dba02928f9c5c5cdd6 is 50, key is test_row_0/C:col10/1731363526172/Put/seqid=0 2024-11-11T22:18:47,445 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/B of 20d39f0a33cd5786c48995acee2197ce into ea9b028355f4464285631557b44e0c63(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:47,445 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:47,445 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/B, priority=13, startTime=1731363527369; duration=0sec 2024-11-11T22:18:47,445 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:47,445 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:B 2024-11-11T22:18:47,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742344_1520 (size=12301) 2024-11-11T22:18:47,457 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ccc67416b16b4b1f8b4f59bd99199fce 2024-11-11T22:18:47,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742345_1521 (size=13153) 2024-11-11T22:18:47,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/2afbc3548e22481987f57750fbbd3efa is 50, key is test_row_0/C:col10/1731363526527/Put/seqid=0 2024-11-11T22:18:47,473 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/50191b5501a744dba02928f9c5c5cdd6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/50191b5501a744dba02928f9c5c5cdd6 2024-11-11T22:18:47,478 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/C of 20d39f0a33cd5786c48995acee2197ce into 50191b5501a744dba02928f9c5c5cdd6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:47,478 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:47,478 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/C, priority=13, startTime=1731363527369; duration=0sec 2024-11-11T22:18:47,479 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:47,479 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:C 2024-11-11T22:18:47,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742346_1522 (size=12301) 2024-11-11T22:18:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:47,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. as already flushing 2024-11-11T22:18:47,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363587717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363587719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363587719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363587720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363587721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363587829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363587829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363587840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363587840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:47,889 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/2afbc3548e22481987f57750fbbd3efa 2024-11-11T22:18:47,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/578d7fabaf1141f6a17fba66adc2072e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/578d7fabaf1141f6a17fba66adc2072e 2024-11-11T22:18:47,895 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/578d7fabaf1141f6a17fba66adc2072e, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T22:18:47,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/ccc67416b16b4b1f8b4f59bd99199fce as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ccc67416b16b4b1f8b4f59bd99199fce 2024-11-11T22:18:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T22:18:47,900 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ccc67416b16b4b1f8b4f59bd99199fce, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T22:18:47,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/2afbc3548e22481987f57750fbbd3efa as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/2afbc3548e22481987f57750fbbd3efa 2024-11-11T22:18:47,904 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/2afbc3548e22481987f57750fbbd3efa, entries=150, sequenceid=377, filesize=12.0 K 2024-11-11T22:18:47,904 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 20d39f0a33cd5786c48995acee2197ce in 532ms, sequenceid=377, compaction requested=false 2024-11-11T22:18:47,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-11T22:18:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-11T22:18:47,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T22:18:47,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1440 sec 2024-11-11T22:18:47,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.1470 sec 2024-11-11T22:18:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:48,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T22:18:48,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:48,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:48,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:48,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:48,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:48,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:48,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f7b5ba14bc8740998c7772aca403bea4 is 50, key is test_row_0/A:col10/1731363528042/Put/seqid=0 2024-11-11T22:18:48,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742347_1523 (size=12297) 2024-11-11T22:18:48,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363588070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363588072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363588072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363588073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363588179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363588190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363588190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363588190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,402 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-11-11T22:18:48,402 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:48,402 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,402 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:48,404 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-11T22:18:48,404 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:48,404 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,404 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,404 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363588399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363588400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363588400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363588400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,406 INFO [Thread-1999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1999. 2024-11-11T22:18:48,406 DEBUG [Thread-1999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:48,406 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,406 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:48,411 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-11T22:18:48,411 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:48,411 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,411 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:48,413 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-11T22:18:48,413 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:48,414 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,414 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:48,414 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:48,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f7b5ba14bc8740998c7772aca403bea4 2024-11-11T22:18:48,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/7e6770a04cfd420a94e4dd18b89c7cd0 is 50, key is test_row_0/B:col10/1731363528042/Put/seqid=0 2024-11-11T22:18:48,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742348_1524 (size=9857) 2024-11-11T22:18:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363588710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363588710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363588710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363588710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44896 deadline: 1731363588740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:48,740 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_0' in 20d39f0a33cd5786c48995acee2197ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:18:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:48,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/7e6770a04cfd420a94e4dd18b89c7cd0 2024-11-11T22:18:48,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/f026131bef324c989d9e2a6b63a12c08 is 50, key is test_row_0/C:col10/1731363528042/Put/seqid=0 2024-11-11T22:18:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742349_1525 (size=9857) 2024-11-11T22:18:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T22:18:48,910 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44932 deadline: 1731363589219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44940 deadline: 1731363589219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44892 deadline: 1731363589219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44872 deadline: 1731363589219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 , the old value is region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., hostname=8d82b89dd27d,36911,1731363387997, seqNum=2 because the exception is null or not the one we care about 2024-11-11T22:18:49,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/f026131bef324c989d9e2a6b63a12c08 2024-11-11T22:18:49,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f7b5ba14bc8740998c7772aca403bea4 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f7b5ba14bc8740998c7772aca403bea4 2024-11-11T22:18:49,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f7b5ba14bc8740998c7772aca403bea4, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T22:18:49,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/7e6770a04cfd420a94e4dd18b89c7cd0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7e6770a04cfd420a94e4dd18b89c7cd0 2024-11-11T22:18:49,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7e6770a04cfd420a94e4dd18b89c7cd0, entries=100, sequenceid=396, filesize=9.6 K 2024-11-11T22:18:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/f026131bef324c989d9e2a6b63a12c08 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/f026131bef324c989d9e2a6b63a12c08 2024-11-11T22:18:49,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/f026131bef324c989d9e2a6b63a12c08, entries=100, sequenceid=396, filesize=9.6 K 2024-11-11T22:18:49,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 20d39f0a33cd5786c48995acee2197ce in 1260ms, sequenceid=396, compaction requested=true 2024-11-11T22:18:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:49,303 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20d39f0a33cd5786c48995acee2197ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:49,303 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:49,304 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:49,304 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:49,304 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/A is initiating minor compaction (all files) 2024-11-11T22:18:49,304 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/B is initiating minor compaction (all files) 2024-11-11T22:18:49,304 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/A in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:49,304 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/B in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:49,304 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ea9b028355f4464285631557b44e0c63, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ccc67416b16b4b1f8b4f59bd99199fce, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7e6770a04cfd420a94e4dd18b89c7cd0] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=34.5 K 2024-11-11T22:18:49,304 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c9eeb5684766420c94b97b791c0d2ad6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/578d7fabaf1141f6a17fba66adc2072e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f7b5ba14bc8740998c7772aca403bea4] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=36.9 K 2024-11-11T22:18:49,304 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ea9b028355f4464285631557b44e0c63, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731363526172 2024-11-11T22:18:49,304 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9eeb5684766420c94b97b791c0d2ad6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731363526172 2024-11-11T22:18:49,305 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 578d7fabaf1141f6a17fba66adc2072e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731363526512 2024-11-11T22:18:49,305 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc67416b16b4b1f8b4f59bd99199fce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731363526512 2024-11-11T22:18:49,305 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b5ba14bc8740998c7772aca403bea4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731363527719 2024-11-11T22:18:49,305 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e6770a04cfd420a94e4dd18b89c7cd0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731363527719 2024-11-11T22:18:49,312 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:49,312 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/5e51547da3c34435b943ed5125a14677 is 50, key is test_row_0/B:col10/1731363528042/Put/seqid=0 2024-11-11T22:18:49,314 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:49,314 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/73149fb21c964a87a01e01d9d30fa139 is 50, key is test_row_0/A:col10/1731363528042/Put/seqid=0 2024-11-11T22:18:49,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742350_1526 (size=13255) 2024-11-11T22:18:49,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742351_1527 (size=13255) 2024-11-11T22:18:49,723 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/5e51547da3c34435b943ed5125a14677 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/5e51547da3c34435b943ed5125a14677 2024-11-11T22:18:49,736 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/73149fb21c964a87a01e01d9d30fa139 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/73149fb21c964a87a01e01d9d30fa139 2024-11-11T22:18:49,736 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/B of 20d39f0a33cd5786c48995acee2197ce into 5e51547da3c34435b943ed5125a14677(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:49,737 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:49,737 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/B, priority=13, startTime=1731363529303; duration=0sec 2024-11-11T22:18:49,737 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:49,737 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:B 2024-11-11T22:18:49,737 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:49,737 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:49,737 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): 20d39f0a33cd5786c48995acee2197ce/C is initiating minor compaction (all files) 2024-11-11T22:18:49,737 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20d39f0a33cd5786c48995acee2197ce/C in TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:49,738 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/50191b5501a744dba02928f9c5c5cdd6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/2afbc3548e22481987f57750fbbd3efa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/f026131bef324c989d9e2a6b63a12c08] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp, totalSize=34.5 K 2024-11-11T22:18:49,738 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 50191b5501a744dba02928f9c5c5cdd6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731363526172 2024-11-11T22:18:49,738 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2afbc3548e22481987f57750fbbd3efa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731363526512 2024-11-11T22:18:49,738 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f026131bef324c989d9e2a6b63a12c08, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731363527719 2024-11-11T22:18:49,740 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/A of 20d39f0a33cd5786c48995acee2197ce into 73149fb21c964a87a01e01d9d30fa139(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:49,740 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:49,740 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/A, priority=13, startTime=1731363529303; duration=0sec 2024-11-11T22:18:49,740 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:49,740 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:A 2024-11-11T22:18:49,743 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20d39f0a33cd5786c48995acee2197ce#C#compaction#455 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:49,744 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/9129184eadd34439b7baccf243f3d904 is 50, key is test_row_0/C:col10/1731363528042/Put/seqid=0 2024-11-11T22:18:49,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742352_1528 (size=13255) 2024-11-11T22:18:50,151 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/9129184eadd34439b7baccf243f3d904 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/9129184eadd34439b7baccf243f3d904 2024-11-11T22:18:50,156 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20d39f0a33cd5786c48995acee2197ce/C of 20d39f0a33cd5786c48995acee2197ce into 9129184eadd34439b7baccf243f3d904(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:50,156 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:50,156 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., storeName=20d39f0a33cd5786c48995acee2197ce/C, priority=13, startTime=1731363529303; duration=0sec 2024-11-11T22:18:50,156 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:50,156 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20d39f0a33cd5786c48995acee2197ce:C 2024-11-11T22:18:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:50,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:18:50,232 INFO [Thread-1986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1986. 2024-11-11T22:18:50,232 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-11-11T22:18:50,232 INFO [Thread-1980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1980. 2024-11-11T22:18:50,232 DEBUG [Thread-1986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:50,232 DEBUG [Thread-1980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:50,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:50,232 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:50,232 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,232 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,232 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:50,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:50,232 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,232 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:50,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:50,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:50,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:50,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:50,233 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:50,234 INFO [Thread-1984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1984. 2024-11-11T22:18:50,234 DEBUG [Thread-1984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:50,234 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,234 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:50,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/00b46860bca54a69ae24d159a302314b is 50, key is test_row_0/A:col10/1731363530230/Put/seqid=0 2024-11-11T22:18:50,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742353_1529 (size=12301) 2024-11-11T22:18:50,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/00b46860bca54a69ae24d159a302314b 2024-11-11T22:18:50,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/94a4e9d18d6346ddb6d135c633a1b74d is 50, key is test_row_0/B:col10/1731363530230/Put/seqid=0 2024-11-11T22:18:50,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742354_1530 (size=12301) 2024-11-11T22:18:50,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/94a4e9d18d6346ddb6d135c633a1b74d 2024-11-11T22:18:50,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/6e665f32cfe2448cac4579cf1f2b938e is 50, key is test_row_0/C:col10/1731363530230/Put/seqid=0 2024-11-11T22:18:50,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742355_1531 (size=12301) 2024-11-11T22:18:50,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/6e665f32cfe2448cac4579cf1f2b938e 2024-11-11T22:18:50,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/00b46860bca54a69ae24d159a302314b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/00b46860bca54a69ae24d159a302314b 2024-11-11T22:18:50,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/00b46860bca54a69ae24d159a302314b, entries=150, sequenceid=418, filesize=12.0 K 2024-11-11T22:18:50,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/94a4e9d18d6346ddb6d135c633a1b74d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94a4e9d18d6346ddb6d135c633a1b74d 2024-11-11T22:18:50,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94a4e9d18d6346ddb6d135c633a1b74d, entries=150, sequenceid=418, filesize=12.0 K 2024-11-11T22:18:50,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/6e665f32cfe2448cac4579cf1f2b938e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/6e665f32cfe2448cac4579cf1f2b938e 2024-11-11T22:18:50,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/6e665f32cfe2448cac4579cf1f2b938e, entries=150, sequenceid=418, filesize=12.0 K 2024-11-11T22:18:50,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 20d39f0a33cd5786c48995acee2197ce in 462ms, sequenceid=418, compaction requested=false 2024-11-11T22:18:50,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20d39f0a33cd5786c48995acee2197ce: 2024-11-11T22:18:50,760 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-11-11T22:18:50,761 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:18:50,761 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,761 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1125 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3375 rows 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1112 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3326 rows 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1132 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3394 rows 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1126 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3378 rows 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1118 2024-11-11T22:18:50,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3354 rows 2024-11-11T22:18:50,761 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:18:50,761 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:18:50,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:50,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:18:50,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:18:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T22:18:50,765 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363530765"}]},"ts":"1731363530765"} 2024-11-11T22:18:50,766 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:18:50,766 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:18:50,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:18:50,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, UNASSIGN}] 2024-11-11T22:18:50,769 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, UNASSIGN 2024-11-11T22:18:50,769 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=20d39f0a33cd5786c48995acee2197ce, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:50,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, UNASSIGN because future has completed 2024-11-11T22:18:50,771 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:18:50,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:18:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T22:18:50,923 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:50,923 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:18:50,923 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 20d39f0a33cd5786c48995acee2197ce, disabling compactions & flushes 2024-11-11T22:18:50,923 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:50,923 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:50,923 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. after waiting 0 ms 2024-11-11T22:18:50,923 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:50,923 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 20d39f0a33cd5786c48995acee2197ce 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-11T22:18:50,924 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=A 2024-11-11T22:18:50,924 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:50,924 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=B 2024-11-11T22:18:50,924 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:50,924 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 20d39f0a33cd5786c48995acee2197ce, store=C 2024-11-11T22:18:50,924 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:50,928 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f1e1457c315e4fe18e4db86ee68b312f is 50, key is test_row_0/A:col10/1731363530759/Put/seqid=0 2024-11-11T22:18:50,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742356_1532 (size=9857) 2024-11-11T22:18:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T22:18:51,333 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f1e1457c315e4fe18e4db86ee68b312f 2024-11-11T22:18:51,338 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/35d51e7037834561ba3d9218c90021d9 is 50, key is test_row_0/B:col10/1731363530759/Put/seqid=0 2024-11-11T22:18:51,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742357_1533 (size=9857) 2024-11-11T22:18:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T22:18:51,742 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/35d51e7037834561ba3d9218c90021d9 2024-11-11T22:18:51,747 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/1375c2f6662546b9b8e73c169df65b38 is 50, key is test_row_0/C:col10/1731363530759/Put/seqid=0 2024-11-11T22:18:51,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742358_1534 (size=9857) 2024-11-11T22:18:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T22:18:52,150 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/1375c2f6662546b9b8e73c169df65b38 2024-11-11T22:18:52,154 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/A/f1e1457c315e4fe18e4db86ee68b312f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f1e1457c315e4fe18e4db86ee68b312f 2024-11-11T22:18:52,156 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f1e1457c315e4fe18e4db86ee68b312f, entries=100, sequenceid=423, filesize=9.6 K 2024-11-11T22:18:52,157 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/B/35d51e7037834561ba3d9218c90021d9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/35d51e7037834561ba3d9218c90021d9 2024-11-11T22:18:52,159 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/35d51e7037834561ba3d9218c90021d9, entries=100, sequenceid=423, filesize=9.6 K 2024-11-11T22:18:52,160 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/.tmp/C/1375c2f6662546b9b8e73c169df65b38 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1375c2f6662546b9b8e73c169df65b38 2024-11-11T22:18:52,162 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1375c2f6662546b9b8e73c169df65b38, entries=100, sequenceid=423, filesize=9.6 K 2024-11-11T22:18:52,163 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 20d39f0a33cd5786c48995acee2197ce in 1240ms, sequenceid=423, compaction requested=true 2024-11-11T22:18:52,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/906af6cd283a474b9d2d46f6c5a18444, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/b067c61538a74e1bb770fdc2832c0454, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b72077a3f074f879590ed064e4c875f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1dbb96a3a98c412ea33b462fa596faf7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/3a25fa886b7248ab9b9068336f4cfeb7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/8efcca8b1f2c40219a43bd9770c115cd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4145711f10a04f1189a5ea09b6b7d121, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/587ce51d65344530990e0e8e40dae58f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4191ba53d21140c493dd7db0f913725b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1a941d7e82814f89a3ac4b0ff1fe6f88, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cfc895638ad94114911e29696377e699, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b847b2d38a04b21bcc32bbc48273802, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/bee82233d6554388990baa98a66df987, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f8a15eb5c59c4a328590c8375a5fd617, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/6071316b646b45a2a27060541dab4033, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c01e0adb0135444285b0f35a38a68aa1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/55e97dd8512647beb0d2e5072fdd3c2f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f205394874854fe2b7f0595b53940b67, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/ff3f33de91504cf0aa7e330a9529d9d4, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/324b4738d7a2435ea320032a3dd37c33, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cc1194d1256a412b837adaae054e7a63, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1f8b1736ebab448da07b4aacd88d8d64, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/32aac6e935844b1d8380e40fd18cd7be, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/a68580c3b4694445a418e30ff501e02f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/71fb5cb271fc4e0fb4fee03c2fdf150b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c9eeb5684766420c94b97b791c0d2ad6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/578d7fabaf1141f6a17fba66adc2072e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f7b5ba14bc8740998c7772aca403bea4] to archive 2024-11-11T22:18:52,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:18:52,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/906af6cd283a474b9d2d46f6c5a18444 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/906af6cd283a474b9d2d46f6c5a18444 2024-11-11T22:18:52,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/b067c61538a74e1bb770fdc2832c0454 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/b067c61538a74e1bb770fdc2832c0454 2024-11-11T22:18:52,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b72077a3f074f879590ed064e4c875f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b72077a3f074f879590ed064e4c875f 2024-11-11T22:18:52,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1dbb96a3a98c412ea33b462fa596faf7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1dbb96a3a98c412ea33b462fa596faf7 2024-11-11T22:18:52,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/3a25fa886b7248ab9b9068336f4cfeb7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/3a25fa886b7248ab9b9068336f4cfeb7 2024-11-11T22:18:52,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/8efcca8b1f2c40219a43bd9770c115cd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/8efcca8b1f2c40219a43bd9770c115cd 2024-11-11T22:18:52,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4145711f10a04f1189a5ea09b6b7d121 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4145711f10a04f1189a5ea09b6b7d121 2024-11-11T22:18:52,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/587ce51d65344530990e0e8e40dae58f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/587ce51d65344530990e0e8e40dae58f 2024-11-11T22:18:52,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4191ba53d21140c493dd7db0f913725b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/4191ba53d21140c493dd7db0f913725b 2024-11-11T22:18:52,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1a941d7e82814f89a3ac4b0ff1fe6f88 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1a941d7e82814f89a3ac4b0ff1fe6f88 2024-11-11T22:18:52,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cfc895638ad94114911e29696377e699 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cfc895638ad94114911e29696377e699 2024-11-11T22:18:52,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b847b2d38a04b21bcc32bbc48273802 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/2b847b2d38a04b21bcc32bbc48273802 2024-11-11T22:18:52,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/bee82233d6554388990baa98a66df987 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/bee82233d6554388990baa98a66df987 2024-11-11T22:18:52,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f8a15eb5c59c4a328590c8375a5fd617 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f8a15eb5c59c4a328590c8375a5fd617 2024-11-11T22:18:52,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/6071316b646b45a2a27060541dab4033 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/6071316b646b45a2a27060541dab4033 2024-11-11T22:18:52,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c01e0adb0135444285b0f35a38a68aa1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c01e0adb0135444285b0f35a38a68aa1 2024-11-11T22:18:52,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/55e97dd8512647beb0d2e5072fdd3c2f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/55e97dd8512647beb0d2e5072fdd3c2f 2024-11-11T22:18:52,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f205394874854fe2b7f0595b53940b67 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f205394874854fe2b7f0595b53940b67 2024-11-11T22:18:52,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/ff3f33de91504cf0aa7e330a9529d9d4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/ff3f33de91504cf0aa7e330a9529d9d4 2024-11-11T22:18:52,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/324b4738d7a2435ea320032a3dd37c33 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/324b4738d7a2435ea320032a3dd37c33 2024-11-11T22:18:52,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cc1194d1256a412b837adaae054e7a63 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/cc1194d1256a412b837adaae054e7a63 2024-11-11T22:18:52,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1f8b1736ebab448da07b4aacd88d8d64 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/1f8b1736ebab448da07b4aacd88d8d64 2024-11-11T22:18:52,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/32aac6e935844b1d8380e40fd18cd7be to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/32aac6e935844b1d8380e40fd18cd7be 2024-11-11T22:18:52,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/a68580c3b4694445a418e30ff501e02f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/a68580c3b4694445a418e30ff501e02f 2024-11-11T22:18:52,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/71fb5cb271fc4e0fb4fee03c2fdf150b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/71fb5cb271fc4e0fb4fee03c2fdf150b 2024-11-11T22:18:52,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c9eeb5684766420c94b97b791c0d2ad6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/c9eeb5684766420c94b97b791c0d2ad6 2024-11-11T22:18:52,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/578d7fabaf1141f6a17fba66adc2072e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/578d7fabaf1141f6a17fba66adc2072e 2024-11-11T22:18:52,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f7b5ba14bc8740998c7772aca403bea4 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f7b5ba14bc8740998c7772aca403bea4 2024-11-11T22:18:52,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d3a1a684b5244469a89cf6409f141b05, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/65dfa58a494e4896be0491abbacdb801, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/662331c4d072420bb530bbdcac85d3df, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dc585f97d6df4b12b73e42dff9850716, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74b449119c5b45c0aa928320ea85abb7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ee4682f49ab241b4a19734433832ae54, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/62c39caaffa4416185010b628241dbe5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7665d78e30854970af8166a98a3d4741, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74786d2fdbac4a0e9205171ed6611507, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/06c469bc6bd9426fa5cc948b925cdb49, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ac69bcff46424846bb40248e2312af1d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d917d0265c264b0fb74f027b91465929, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dcc272b2e1914bc28073b3f8aa90b71b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8afb980412214a9cbc5389b2b4b0e722, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/2f76f5c4646a4c6ebd610f1cf828c8fa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/76d298e1f7494bddb476314b793b12de, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d5da66c367a1407da4ef116412181fbd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94f47229c55f4e94a1793cea28b2fb1a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/eb071683743a4494952b8f99262121e5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/b07fee7a37564c269e71505997870732, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a625ae7e021c42c2ba5cf25b5bec4791, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/0d9f8f40ea0f4c3799532650c0a1514e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8cdf4c60e0134a79ba7bb083ec3cba2f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a8407518b194492b96808c73444638ba, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ea9b028355f4464285631557b44e0c63, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d6e9b4ccf5484a21aa2d3498055d43ab, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ccc67416b16b4b1f8b4f59bd99199fce, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7e6770a04cfd420a94e4dd18b89c7cd0] to archive 2024-11-11T22:18:52,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:18:52,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d3a1a684b5244469a89cf6409f141b05 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d3a1a684b5244469a89cf6409f141b05 2024-11-11T22:18:52,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/65dfa58a494e4896be0491abbacdb801 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/65dfa58a494e4896be0491abbacdb801 2024-11-11T22:18:52,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/662331c4d072420bb530bbdcac85d3df to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/662331c4d072420bb530bbdcac85d3df 2024-11-11T22:18:52,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dc585f97d6df4b12b73e42dff9850716 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dc585f97d6df4b12b73e42dff9850716 2024-11-11T22:18:52,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74b449119c5b45c0aa928320ea85abb7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74b449119c5b45c0aa928320ea85abb7 2024-11-11T22:18:52,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ee4682f49ab241b4a19734433832ae54 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ee4682f49ab241b4a19734433832ae54 2024-11-11T22:18:52,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/62c39caaffa4416185010b628241dbe5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/62c39caaffa4416185010b628241dbe5 2024-11-11T22:18:52,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7665d78e30854970af8166a98a3d4741 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7665d78e30854970af8166a98a3d4741 2024-11-11T22:18:52,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74786d2fdbac4a0e9205171ed6611507 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/74786d2fdbac4a0e9205171ed6611507 2024-11-11T22:18:52,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/06c469bc6bd9426fa5cc948b925cdb49 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/06c469bc6bd9426fa5cc948b925cdb49 2024-11-11T22:18:52,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ac69bcff46424846bb40248e2312af1d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ac69bcff46424846bb40248e2312af1d 2024-11-11T22:18:52,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d917d0265c264b0fb74f027b91465929 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d917d0265c264b0fb74f027b91465929 2024-11-11T22:18:52,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dcc272b2e1914bc28073b3f8aa90b71b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/dcc272b2e1914bc28073b3f8aa90b71b 2024-11-11T22:18:52,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8afb980412214a9cbc5389b2b4b0e722 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8afb980412214a9cbc5389b2b4b0e722 2024-11-11T22:18:52,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/2f76f5c4646a4c6ebd610f1cf828c8fa to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/2f76f5c4646a4c6ebd610f1cf828c8fa 2024-11-11T22:18:52,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/76d298e1f7494bddb476314b793b12de to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/76d298e1f7494bddb476314b793b12de 2024-11-11T22:18:52,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d5da66c367a1407da4ef116412181fbd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d5da66c367a1407da4ef116412181fbd 2024-11-11T22:18:52,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94f47229c55f4e94a1793cea28b2fb1a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94f47229c55f4e94a1793cea28b2fb1a 2024-11-11T22:18:52,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/eb071683743a4494952b8f99262121e5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/eb071683743a4494952b8f99262121e5 2024-11-11T22:18:52,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/b07fee7a37564c269e71505997870732 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/b07fee7a37564c269e71505997870732 2024-11-11T22:18:52,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a625ae7e021c42c2ba5cf25b5bec4791 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a625ae7e021c42c2ba5cf25b5bec4791 2024-11-11T22:18:52,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/0d9f8f40ea0f4c3799532650c0a1514e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/0d9f8f40ea0f4c3799532650c0a1514e 2024-11-11T22:18:52,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8cdf4c60e0134a79ba7bb083ec3cba2f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/8cdf4c60e0134a79ba7bb083ec3cba2f 2024-11-11T22:18:52,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a8407518b194492b96808c73444638ba to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/a8407518b194492b96808c73444638ba 2024-11-11T22:18:52,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ea9b028355f4464285631557b44e0c63 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ea9b028355f4464285631557b44e0c63 2024-11-11T22:18:52,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d6e9b4ccf5484a21aa2d3498055d43ab to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/d6e9b4ccf5484a21aa2d3498055d43ab 2024-11-11T22:18:52,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ccc67416b16b4b1f8b4f59bd99199fce to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/ccc67416b16b4b1f8b4f59bd99199fce 2024-11-11T22:18:52,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7e6770a04cfd420a94e4dd18b89c7cd0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/7e6770a04cfd420a94e4dd18b89c7cd0 2024-11-11T22:18:52,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/5d29adc6140e4a6aa81dd0fccb0c3eca, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/54172755e06f45b98f53b410d6dafdb5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/53fb5ee6d1d24f36a24a5c72bbd6239c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4b4a404e9f2a4b40be292f0ad8d93807, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/77fc97b10c5f4ce68cb2a8a38cbf5416, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/70e2e868dfa748be83f409127dcbdf37, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/cf92d8711bd64d79bbc8c096b54137ad, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/fa029df98ff842b48abf4bb4949ce166, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19b8faf5e621431aad51e369e617f615, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4ebbce5ad77d4bb6bd80d7291c206366, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e14fdea8fcc946f9b7076b3ec73af30a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19f82eae497546beb0bf57b34ec914fb, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/7e922d3c26d2457f92c6775993c4f2ef, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/040131083a9f4acd8fe0895d8ee0e94d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/821e7395513f4812bd92032f3bf149fd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/458a9575f43543d48887af94c6804aef, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/17922ef889074c7287d1ef1e39ad6ad1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0fb83b14a1dc4c8484d329ed09b7fc87, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e6c5c92da52d4e8cb3725889a6d09e7a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1912f51d1c17417f9570b5d887ee75ea, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e83c4370ba7c477e8a4fab9765e6110d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0e8eab0030d04e4ca63fa44c87d5057e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/d910bcba203c454d96b325e65cb0d3f9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/00d6e82b3fdd4bba928834734a7d1064, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/50191b5501a744dba02928f9c5c5cdd6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/bb6eeed4af5a4395bdb2ef8767770c7d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/2afbc3548e22481987f57750fbbd3efa, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/f026131bef324c989d9e2a6b63a12c08] to archive 2024-11-11T22:18:52,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:18:52,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/5d29adc6140e4a6aa81dd0fccb0c3eca to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/5d29adc6140e4a6aa81dd0fccb0c3eca 2024-11-11T22:18:52,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/54172755e06f45b98f53b410d6dafdb5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/54172755e06f45b98f53b410d6dafdb5 2024-11-11T22:18:52,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/53fb5ee6d1d24f36a24a5c72bbd6239c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/53fb5ee6d1d24f36a24a5c72bbd6239c 2024-11-11T22:18:52,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4b4a404e9f2a4b40be292f0ad8d93807 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4b4a404e9f2a4b40be292f0ad8d93807 2024-11-11T22:18:52,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/77fc97b10c5f4ce68cb2a8a38cbf5416 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/77fc97b10c5f4ce68cb2a8a38cbf5416 2024-11-11T22:18:52,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/70e2e868dfa748be83f409127dcbdf37 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/70e2e868dfa748be83f409127dcbdf37 2024-11-11T22:18:52,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/cf92d8711bd64d79bbc8c096b54137ad to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/cf92d8711bd64d79bbc8c096b54137ad 2024-11-11T22:18:52,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/fa029df98ff842b48abf4bb4949ce166 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/fa029df98ff842b48abf4bb4949ce166 2024-11-11T22:18:52,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19b8faf5e621431aad51e369e617f615 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19b8faf5e621431aad51e369e617f615 2024-11-11T22:18:52,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4ebbce5ad77d4bb6bd80d7291c206366 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/4ebbce5ad77d4bb6bd80d7291c206366 2024-11-11T22:18:52,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e14fdea8fcc946f9b7076b3ec73af30a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e14fdea8fcc946f9b7076b3ec73af30a 2024-11-11T22:18:52,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19f82eae497546beb0bf57b34ec914fb to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/19f82eae497546beb0bf57b34ec914fb 2024-11-11T22:18:52,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/7e922d3c26d2457f92c6775993c4f2ef to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/7e922d3c26d2457f92c6775993c4f2ef 2024-11-11T22:18:52,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/040131083a9f4acd8fe0895d8ee0e94d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/040131083a9f4acd8fe0895d8ee0e94d 2024-11-11T22:18:52,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/821e7395513f4812bd92032f3bf149fd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/821e7395513f4812bd92032f3bf149fd 2024-11-11T22:18:52,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/458a9575f43543d48887af94c6804aef to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/458a9575f43543d48887af94c6804aef 2024-11-11T22:18:52,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/17922ef889074c7287d1ef1e39ad6ad1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/17922ef889074c7287d1ef1e39ad6ad1 2024-11-11T22:18:52,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0fb83b14a1dc4c8484d329ed09b7fc87 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0fb83b14a1dc4c8484d329ed09b7fc87 2024-11-11T22:18:52,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e6c5c92da52d4e8cb3725889a6d09e7a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e6c5c92da52d4e8cb3725889a6d09e7a 2024-11-11T22:18:52,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1912f51d1c17417f9570b5d887ee75ea to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1912f51d1c17417f9570b5d887ee75ea 2024-11-11T22:18:52,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e83c4370ba7c477e8a4fab9765e6110d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/e83c4370ba7c477e8a4fab9765e6110d 2024-11-11T22:18:52,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0e8eab0030d04e4ca63fa44c87d5057e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/0e8eab0030d04e4ca63fa44c87d5057e 2024-11-11T22:18:52,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/d910bcba203c454d96b325e65cb0d3f9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/d910bcba203c454d96b325e65cb0d3f9 2024-11-11T22:18:52,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/00d6e82b3fdd4bba928834734a7d1064 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/00d6e82b3fdd4bba928834734a7d1064 2024-11-11T22:18:52,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/50191b5501a744dba02928f9c5c5cdd6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/50191b5501a744dba02928f9c5c5cdd6 2024-11-11T22:18:52,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/bb6eeed4af5a4395bdb2ef8767770c7d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/bb6eeed4af5a4395bdb2ef8767770c7d 2024-11-11T22:18:52,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/2afbc3548e22481987f57750fbbd3efa to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/2afbc3548e22481987f57750fbbd3efa 2024-11-11T22:18:52,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/f026131bef324c989d9e2a6b63a12c08 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/f026131bef324c989d9e2a6b63a12c08 2024-11-11T22:18:52,226 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/recovered.edits/426.seqid, newMaxSeqId=426, maxSeqId=1 2024-11-11T22:18:52,227 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. 2024-11-11T22:18:52,227 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 20d39f0a33cd5786c48995acee2197ce: Waiting for close lock at 1731363530923Running coprocessor pre-close hooks at 1731363530923Disabling compacts and flushes for region at 1731363530923Disabling writes for close at 1731363530923Obtaining lock to block concurrent updates at 1731363530923Preparing flush snapshotting stores in 20d39f0a33cd5786c48995acee2197ce at 1731363530923Finished memstore snapshotting TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731363530924 (+1 ms)Flushing stores of TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce. at 1731363530924Flushing 20d39f0a33cd5786c48995acee2197ce/A: creating writer at 1731363530924Flushing 20d39f0a33cd5786c48995acee2197ce/A: appending metadata at 1731363530927 (+3 ms)Flushing 20d39f0a33cd5786c48995acee2197ce/A: closing flushed file at 1731363530927Flushing 20d39f0a33cd5786c48995acee2197ce/B: creating writer at 1731363531336 (+409 ms)Flushing 20d39f0a33cd5786c48995acee2197ce/B: appending metadata at 1731363531338 (+2 ms)Flushing 20d39f0a33cd5786c48995acee2197ce/B: closing flushed file at 1731363531338Flushing 20d39f0a33cd5786c48995acee2197ce/C: creating writer at 1731363531745 (+407 ms)Flushing 20d39f0a33cd5786c48995acee2197ce/C: appending metadata at 1731363531747 (+2 ms)Flushing 20d39f0a33cd5786c48995acee2197ce/C: closing flushed file at 1731363531747Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f4369f7: reopening flushed file at 1731363532153 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ba1a7e6: reopening flushed file at 1731363532156 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48636c23: reopening flushed file at 1731363532159 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 20d39f0a33cd5786c48995acee2197ce in 1240ms, sequenceid=423, compaction requested=true at 1731363532163 (+4 ms)Writing region close event to WAL at 1731363532224 (+61 ms)Running coprocessor post-close hooks at 1731363532227 (+3 ms)Closed at 1731363532227 2024-11-11T22:18:52,228 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:52,229 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=20d39f0a33cd5786c48995acee2197ce, regionState=CLOSED 2024-11-11T22:18:52,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:18:52,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T22:18:52,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 20d39f0a33cd5786c48995acee2197ce, server=8d82b89dd27d,36911,1731363387997 in 1.4600 sec 2024-11-11T22:18:52,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-11T22:18:52,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20d39f0a33cd5786c48995acee2197ce, UNASSIGN in 1.4650 sec 2024-11-11T22:18:52,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T22:18:52,235 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4670 sec 2024-11-11T22:18:52,236 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363532236"}]},"ts":"1731363532236"} 2024-11-11T22:18:52,237 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:18:52,237 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:18:52,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4750 sec 2024-11-11T22:18:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T22:18:52,909 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:52,909 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:18:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:52,911 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T22:18:52,911 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:52,913 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:52,915 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/recovered.edits] 2024-11-11T22:18:52,917 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/00b46860bca54a69ae24d159a302314b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/00b46860bca54a69ae24d159a302314b 2024-11-11T22:18:52,918 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/73149fb21c964a87a01e01d9d30fa139 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/73149fb21c964a87a01e01d9d30fa139 2024-11-11T22:18:52,919 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f1e1457c315e4fe18e4db86ee68b312f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/A/f1e1457c315e4fe18e4db86ee68b312f 2024-11-11T22:18:52,921 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/35d51e7037834561ba3d9218c90021d9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/35d51e7037834561ba3d9218c90021d9 2024-11-11T22:18:52,922 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/5e51547da3c34435b943ed5125a14677 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/5e51547da3c34435b943ed5125a14677 2024-11-11T22:18:52,922 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94a4e9d18d6346ddb6d135c633a1b74d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/B/94a4e9d18d6346ddb6d135c633a1b74d 2024-11-11T22:18:52,924 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1375c2f6662546b9b8e73c169df65b38 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/1375c2f6662546b9b8e73c169df65b38 2024-11-11T22:18:52,924 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/6e665f32cfe2448cac4579cf1f2b938e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/6e665f32cfe2448cac4579cf1f2b938e 2024-11-11T22:18:52,925 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/9129184eadd34439b7baccf243f3d904 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/C/9129184eadd34439b7baccf243f3d904 2024-11-11T22:18:52,926 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/recovered.edits/426.seqid to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce/recovered.edits/426.seqid 2024-11-11T22:18:52,927 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/20d39f0a33cd5786c48995acee2197ce 2024-11-11T22:18:52,927 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:18:52,928 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:52,929 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:18:52,931 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:18:52,931 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:52,932 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:18:52,932 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731363532932"}]},"ts":"9223372036854775807"} 2024-11-11T22:18:52,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:18:52,933 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 20d39f0a33cd5786c48995acee2197ce, NAME => 'TestAcidGuarantees,,1731363506177.20d39f0a33cd5786c48995acee2197ce.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:18:52,933 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:18:52,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731363532933"}]},"ts":"9223372036854775807"} 2024-11-11T22:18:52,934 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:18:52,934 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:52,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 25 msec 2024-11-11T22:18:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T22:18:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:18:53,019 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:53,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=578 (was 658), ProcessCount=11 (was 11), AvailableMemoryMB=878 (was 1623) 2024-11-11T22:18:53,034 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=578, ProcessCount=11, AvailableMemoryMB=878 2024-11-11T22:18:53,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:18:53,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:18:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:53,037 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T22:18:53,037 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:53,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-11T22:18:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:18:53,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T22:18:53,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742359_1535 (size=1061) 2024-11-11T22:18:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:18:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:18:53,444 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e 2024-11-11T22:18:53,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742360_1536 (size=53) 2024-11-11T22:18:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:18:53,848 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:18:53,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e0c62c1438a42621889aef2f44898dc9, disabling compactions & flushes 2024-11-11T22:18:53,849 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:53,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:53,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. after waiting 0 ms 2024-11-11T22:18:53,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:53,849 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:53,849 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e0c62c1438a42621889aef2f44898dc9: Waiting for close lock at 1731363533849Disabling compacts and flushes for region at 1731363533849Disabling writes for close at 1731363533849Writing region close event to WAL at 1731363533849Closed at 1731363533849 2024-11-11T22:18:53,850 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T22:18:53,850 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731363533850"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731363533850"}]},"ts":"1731363533850"} 2024-11-11T22:18:53,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T22:18:53,852 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T22:18:53,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363533852"}]},"ts":"1731363533852"} 2024-11-11T22:18:53,853 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T22:18:53,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, ASSIGN}] 2024-11-11T22:18:53,854 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, ASSIGN 2024-11-11T22:18:53,855 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, ASSIGN; state=OFFLINE, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=false 2024-11-11T22:18:54,006 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=e0c62c1438a42621889aef2f44898dc9, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:54,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, ASSIGN because future has completed 2024-11-11T22:18:54,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:18:54,161 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:54,161 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:18:54,161 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,161 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:18:54,161 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,161 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,162 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,163 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:54,163 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c62c1438a42621889aef2f44898dc9 columnFamilyName A 2024-11-11T22:18:54,163 DEBUG [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:54,164 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(327): Store=e0c62c1438a42621889aef2f44898dc9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:54,164 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,164 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:54,165 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c62c1438a42621889aef2f44898dc9 columnFamilyName B 2024-11-11T22:18:54,165 DEBUG [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:54,165 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(327): Store=e0c62c1438a42621889aef2f44898dc9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:54,165 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,166 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:54,166 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c62c1438a42621889aef2f44898dc9 columnFamilyName C 2024-11-11T22:18:54,166 DEBUG [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:54,166 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(327): Store=e0c62c1438a42621889aef2f44898dc9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:54,166 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:54,166 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,167 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,167 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,168 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,168 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,168 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:18:54,169 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:18:54,170 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T22:18:54,171 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened e0c62c1438a42621889aef2f44898dc9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58983930, jitterRate=-0.12107095122337341}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:18:54,171 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:54,171 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for e0c62c1438a42621889aef2f44898dc9: Running coprocessor pre-open hook at 1731363534161Writing region info on filesystem at 1731363534161Initializing all the Stores at 1731363534162 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363534162Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363534162Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363534162Cleaning up temporary data from old regions at 1731363534168 (+6 ms)Running coprocessor post-open hooks at 1731363534171 (+3 ms)Region opened successfully at 1731363534171 2024-11-11T22:18:54,172 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., pid=144, masterSystemTime=1731363534159 2024-11-11T22:18:54,173 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:54,173 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:54,174 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=e0c62c1438a42621889aef2f44898dc9, regionState=OPEN, openSeqNum=2, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:54,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:18:54,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-11T22:18:54,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 in 168 msec 2024-11-11T22:18:54,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-11T22:18:54,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, ASSIGN in 324 msec 2024-11-11T22:18:54,178 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T22:18:54,179 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363534178"}]},"ts":"1731363534178"} 2024-11-11T22:18:54,179 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T22:18:54,180 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T22:18:54,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-11T22:18:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T22:18:55,179 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:55,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f71979f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:55,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:55,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:55,181 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c42a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:55,182 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e5b24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:55,183 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:55,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:55,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T22:18:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 8d82b89dd27d,41919,1731363387169 2024-11-11T22:18:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4cccd868 2024-11-11T22:18:55,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T22:18:55,186 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T22:18:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T22:18:55,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T22:18:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T22:18:55,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742361_1537 (size=1097) 2024-11-11T22:18:55,597 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T22:18:55,597 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T22:18:55,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:18:55,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, REOPEN/MOVE}] 2024-11-11T22:18:55,602 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, REOPEN/MOVE 2024-11-11T22:18:55,603 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e0c62c1438a42621889aef2f44898dc9, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:55,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, REOPEN/MOVE because future has completed 2024-11-11T22:18:55,604 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:18:55,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:18:55,756 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:55,756 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:18:55,756 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing e0c62c1438a42621889aef2f44898dc9, disabling compactions & flushes 2024-11-11T22:18:55,756 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:55,756 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:55,756 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. after waiting 0 ms 2024-11-11T22:18:55,756 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:55,759 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T22:18:55,760 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:55,760 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for e0c62c1438a42621889aef2f44898dc9: Waiting for close lock at 1731363535756Running coprocessor pre-close hooks at 1731363535756Disabling compacts and flushes for region at 1731363535756Disabling writes for close at 1731363535756Writing region close event to WAL at 1731363535757 (+1 ms)Running coprocessor post-close hooks at 1731363535760 (+3 ms)Closed at 1731363535760 2024-11-11T22:18:55,760 WARN [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: e0c62c1438a42621889aef2f44898dc9 to self. 2024-11-11T22:18:55,761 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:55,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e0c62c1438a42621889aef2f44898dc9, regionState=CLOSED 2024-11-11T22:18:55,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:18:55,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-11T22:18:55,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 in 159 msec 2024-11-11T22:18:55,765 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, REOPEN/MOVE; state=CLOSED, location=8d82b89dd27d,36911,1731363387997; forceNewPlan=false, retain=true 2024-11-11T22:18:55,916 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e0c62c1438a42621889aef2f44898dc9, regionState=OPENING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:55,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, REOPEN/MOVE because future has completed 2024-11-11T22:18:55,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:18:56,071 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,071 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} 2024-11-11T22:18:56,071 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,071 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T22:18:56,072 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,072 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,073 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,073 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:56,073 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c62c1438a42621889aef2f44898dc9 columnFamilyName A 2024-11-11T22:18:56,074 DEBUG [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:56,074 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(327): Store=e0c62c1438a42621889aef2f44898dc9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:56,075 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,075 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:56,075 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c62c1438a42621889aef2f44898dc9 columnFamilyName B 2024-11-11T22:18:56,075 DEBUG [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:56,076 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(327): Store=e0c62c1438a42621889aef2f44898dc9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:56,076 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,076 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T22:18:56,076 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0c62c1438a42621889aef2f44898dc9 columnFamilyName C 2024-11-11T22:18:56,076 DEBUG [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:56,077 INFO [StoreOpener-e0c62c1438a42621889aef2f44898dc9-1 {}] regionserver.HStore(327): Store=e0c62c1438a42621889aef2f44898dc9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T22:18:56,077 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,077 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,077 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,078 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,079 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,079 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,079 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T22:18:56,080 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,080 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened e0c62c1438a42621889aef2f44898dc9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68182687, jitterRate=0.016001209616661072}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T22:18:56,080 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,081 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for e0c62c1438a42621889aef2f44898dc9: Running coprocessor pre-open hook at 1731363536072Writing region info on filesystem at 1731363536072Initializing all the Stores at 1731363536072Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363536072Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363536072Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731363536072Cleaning up temporary data from old regions at 1731363536079 (+7 ms)Running coprocessor post-open hooks at 1731363536080 (+1 ms)Region opened successfully at 1731363536081 (+1 ms) 2024-11-11T22:18:56,081 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., pid=149, masterSystemTime=1731363536069 2024-11-11T22:18:56,082 DEBUG [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,082 INFO [RS_OPEN_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,083 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e0c62c1438a42621889aef2f44898dc9, regionState=OPEN, openSeqNum=5, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:18:56,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-11T22:18:56,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 in 167 msec 2024-11-11T22:18:56,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T22:18:56,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, REOPEN/MOVE in 484 msec 2024-11-11T22:18:56,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-11T22:18:56,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-11T22:18:56,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-11-11T22:18:56,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T22:18:56,091 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:56,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7663d9ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,092 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28784c10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,093 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5e5660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876a305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,095 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a953e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,096 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15360311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6266891e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,097 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e30d861, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,098 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e0bd97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,099 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79538ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,099 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,099 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,099 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f688b8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,101 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4550b0c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4601a12f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,102 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9906a91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,103 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c53fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d16a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,104 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9e3586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab3a120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16487325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,107 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3290abab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,108 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64956c4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5129a55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,110 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,111 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec5d760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,112 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f54f32b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f0ee4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,113 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,113 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64063f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,114 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606f1a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf3480a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8d82b89dd27d,41919,-1 for getting cluster id 2024-11-11T22:18:56,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T22:18:56,116 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '804d1099-6056-4e2e-b24a-46ac147d80e8' 2024-11-11T22:18:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T22:18:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "804d1099-6056-4e2e-b24a-46ac147d80e8" 2024-11-11T22:18:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb5afab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8d82b89dd27d,41919,-1] 2024-11-11T22:18:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T22:18:56,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:18:56,116 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T22:18:56,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64eead59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T22:18:56,117 DEBUG [Thread-2392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,117 DEBUG [Thread-2396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,120 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,120 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,121 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,122 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-11T22:18:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,123 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:18:56,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,123 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:56,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:56,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,128 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,129 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T22:18:56,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:18:56,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:56,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:18:56,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:56,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:18:56,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,136 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,142 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363536138, max timestamp=1731363536153Num uniques 859; 2024-11-11T22:18:56,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731363536138, max timestamp=1731363536153Num uniques 859; 2024-11-11T22:18:56,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731363536138, max timestamp=1731363536153Num uniques 859; 2024-11-11T22:18:56,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:18:56,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T22:18:56,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8d82b89dd27d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T22:18:56,168 DEBUG [Thread-2394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T22:18:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8d82b89dd27d,36911,1731363387997, seqNum=-1] 2024-11-11T22:18:56,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111ee83789a8074209a886ce1bf4b0c91d_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363536130/Put/seqid=0 2024-11-11T22:18:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T22:18:56,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T22:18:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5] 2024-11-11T22:18:56,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742362_1538 (size=12154) 2024-11-11T22:18:56,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:56,184 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111ee83789a8074209a886ce1bf4b0c91d_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111ee83789a8074209a886ce1bf4b0c91d_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363596176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363596177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363596180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,185 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1e8c97566131481e8860c46b65b8dd77, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1e8c97566131481e8860c46b65b8dd77 is 175, key is test_row_0/A:col10/1731363536130/Put/seqid=0 2024-11-11T22:18:56,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363596181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742363_1539 (size=30955) 2024-11-11T22:18:56,189 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1e8c97566131481e8860c46b65b8dd77 2024-11-11T22:18:56,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363596185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1615484a4df44134b5fe908061cdd34a is 50, key is test_row_0/B:col10/1731363536130/Put/seqid=0 2024-11-11T22:18:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:18:56,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742364_1540 (size=12001) 2024-11-11T22:18:56,250 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T22:18:56,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T22:18:56,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:18:56,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363596289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363596290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363596292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363596300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363596300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T22:18:56,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:18:56,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:18:56,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363596499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363596499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363596499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363596509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363596510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T22:18:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:18:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:56,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1615484a4df44134b5fe908061cdd34a 2024-11-11T22:18:56,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/86801e96e01046029baff540acd601ec is 50, key is test_row_0/C:col10/1731363536130/Put/seqid=0 2024-11-11T22:18:56,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742365_1541 (size=12001) 2024-11-11T22:18:56,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/86801e96e01046029baff540acd601ec 2024-11-11T22:18:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1e8c97566131481e8860c46b65b8dd77 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e8c97566131481e8860c46b65b8dd77 2024-11-11T22:18:56,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e8c97566131481e8860c46b65b8dd77, entries=150, sequenceid=14, filesize=30.2 K 2024-11-11T22:18:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1615484a4df44134b5fe908061cdd34a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1615484a4df44134b5fe908061cdd34a 2024-11-11T22:18:56,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1615484a4df44134b5fe908061cdd34a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T22:18:56,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/86801e96e01046029baff540acd601ec as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/86801e96e01046029baff540acd601ec 2024-11-11T22:18:56,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/86801e96e01046029baff540acd601ec, entries=150, sequenceid=14, filesize=11.7 K 2024-11-11T22:18:56,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for e0c62c1438a42621889aef2f44898dc9 in 569ms, sequenceid=14, compaction requested=false 2024-11-11T22:18:56,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:18:56,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-11T22:18:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:56,747 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=136.42 KB heapSize=380.69 KB 2024-11-11T22:18:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:18:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:18:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:18:56,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:18:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:18:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111112043c3c8cd3439f94156a36dd5279c0_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363536182/Put/seqid=0 2024-11-11T22:18:56,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742366_1542 (size=12154) 2024-11-11T22:18:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:56,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:18:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363596822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363596822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363596823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363596823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363596827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363596929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363596940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363596940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363597139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363597149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363597150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:57,160 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111112043c3c8cd3439f94156a36dd5279c0_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111112043c3c8cd3439f94156a36dd5279c0_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:57,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/272425726c454dfb82a760557de16dd8, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:18:57,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/272425726c454dfb82a760557de16dd8 is 175, key is test_row_0/A:col10/1731363536182/Put/seqid=0 2024-11-11T22:18:57,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742367_1543 (size=30955) 2024-11-11T22:18:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:18:57,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363597339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363597339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363597449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363597459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363597459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,564 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/272425726c454dfb82a760557de16dd8 2024-11-11T22:18:57,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1e9a3297e14744749000f2811ac90007 is 50, key is test_row_0/B:col10/1731363536182/Put/seqid=0 2024-11-11T22:18:57,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742368_1544 (size=12001) 2024-11-11T22:18:57,603 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1e9a3297e14744749000f2811ac90007 2024-11-11T22:18:57,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/69b98701e25b4685a063e874f1a3d9f1 is 50, key is test_row_0/C:col10/1731363536182/Put/seqid=0 2024-11-11T22:18:57,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742369_1545 (size=12001) 2024-11-11T22:18:57,743 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T22:18:57,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363597959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363597969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:57,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363597969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:58,046 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/69b98701e25b4685a063e874f1a3d9f1 2024-11-11T22:18:58,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/272425726c454dfb82a760557de16dd8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/272425726c454dfb82a760557de16dd8 2024-11-11T22:18:58,052 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/272425726c454dfb82a760557de16dd8, entries=150, sequenceid=49, filesize=30.2 K 2024-11-11T22:18:58,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1e9a3297e14744749000f2811ac90007 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1e9a3297e14744749000f2811ac90007 2024-11-11T22:18:58,057 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1e9a3297e14744749000f2811ac90007, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T22:18:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/69b98701e25b4685a063e874f1a3d9f1 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/69b98701e25b4685a063e874f1a3d9f1 2024-11-11T22:18:58,062 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/69b98701e25b4685a063e874f1a3d9f1, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T22:18:58,063 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~380.64 KB/389776, currentSize=46.96 KB/48090 for e0c62c1438a42621889aef2f44898dc9 in 1316ms, sequenceid=49, compaction requested=false 2024-11-11T22:18:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:18:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-11T22:18:58,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-11T22:18:58,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T22:18:58,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-11T22:18:58,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.9460 sec 2024-11-11T22:18:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T22:18:58,269 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:18:58,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:18:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-11T22:18:58,271 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:18:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:18:58,272 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:18:58,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:18:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:58,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T22:18:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:18:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:18:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:18:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:58,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111124fc5ca344549deb495afb93b7f0c4a_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363538349/Put/seqid=0 2024-11-11T22:18:58,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742370_1546 (size=17034) 2024-11-11T22:18:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:18:58,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363598395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:58,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363598397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:58,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:58,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:18:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:18:58,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363598509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:58,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363598509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:58,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:18:58,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:18:58,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:18:58,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363598719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:58,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363598720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:58,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:18:58,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:18:58,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:18:58,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:58,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111124fc5ca344549deb495afb93b7f0c4a_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111124fc5ca344549deb495afb93b7f0c4a_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:58,765 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/663a7343cf1c4ea6838bbdbd377c9084, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:18:58,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/663a7343cf1c4ea6838bbdbd377c9084 is 175, key is test_row_0/A:col10/1731363538349/Put/seqid=0 2024-11-11T22:18:58,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742371_1547 (size=48139) 2024-11-11T22:18:58,770 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/663a7343cf1c4ea6838bbdbd377c9084 2024-11-11T22:18:58,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/aca9d746ab8b42c189180f71951b66a0 is 50, key is test_row_0/B:col10/1731363538349/Put/seqid=0 2024-11-11T22:18:58,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742372_1548 (size=12001) 2024-11-11T22:18:58,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/aca9d746ab8b42c189180f71951b66a0 2024-11-11T22:18:58,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/a84fb3043ac4416785dfede8ccc8dd3d is 50, key is test_row_0/C:col10/1731363538349/Put/seqid=0 2024-11-11T22:18:58,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742373_1549 (size=12001) 2024-11-11T22:18:58,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/a84fb3043ac4416785dfede8ccc8dd3d 2024-11-11T22:18:58,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/663a7343cf1c4ea6838bbdbd377c9084 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/663a7343cf1c4ea6838bbdbd377c9084 2024-11-11T22:18:58,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/663a7343cf1c4ea6838bbdbd377c9084, entries=250, sequenceid=61, filesize=47.0 K 2024-11-11T22:18:58,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/aca9d746ab8b42c189180f71951b66a0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/aca9d746ab8b42c189180f71951b66a0 2024-11-11T22:18:58,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/aca9d746ab8b42c189180f71951b66a0, entries=150, sequenceid=61, filesize=11.7 K 2024-11-11T22:18:58,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/a84fb3043ac4416785dfede8ccc8dd3d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a84fb3043ac4416785dfede8ccc8dd3d 2024-11-11T22:18:58,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a84fb3043ac4416785dfede8ccc8dd3d, entries=150, sequenceid=61, filesize=11.7 K 2024-11-11T22:18:58,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e0c62c1438a42621889aef2f44898dc9 in 499ms, sequenceid=61, compaction requested=true 2024-11-11T22:18:58,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:18:58,849 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:58,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:18:58,850 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:58,850 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/A is initiating minor compaction (all files) 2024-11-11T22:18:58,850 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/A in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,850 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e8c97566131481e8860c46b65b8dd77, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/272425726c454dfb82a760557de16dd8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/663a7343cf1c4ea6838bbdbd377c9084] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=107.5 K 2024-11-11T22:18:58,850 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,850 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e8c97566131481e8860c46b65b8dd77, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/272425726c454dfb82a760557de16dd8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/663a7343cf1c4ea6838bbdbd377c9084] 2024-11-11T22:18:58,851 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e8c97566131481e8860c46b65b8dd77, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363536124 2024-11-11T22:18:58,851 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272425726c454dfb82a760557de16dd8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731363536174 2024-11-11T22:18:58,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:58,851 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 663a7343cf1c4ea6838bbdbd377c9084, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731363536816 2024-11-11T22:18:58,852 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:58,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:18:58,853 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:58,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:58,853 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/B is initiating minor compaction (all files) 2024-11-11T22:18:58,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:18:58,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:58,853 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/B in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,853 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1615484a4df44134b5fe908061cdd34a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1e9a3297e14744749000f2811ac90007, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/aca9d746ab8b42c189180f71951b66a0] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=35.2 K 2024-11-11T22:18:58,853 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1615484a4df44134b5fe908061cdd34a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363536124 2024-11-11T22:18:58,854 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e9a3297e14744749000f2811ac90007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731363536174 2024-11-11T22:18:58,854 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting aca9d746ab8b42c189180f71951b66a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731363536826 2024-11-11T22:18:58,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-11T22:18:58,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:58,883 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:18:58,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:18:58,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:58,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:18:58,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:58,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:18:58,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:18:58,889 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:18:58,897 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#B#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:58,898 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111da515500b00149708f150b5717c30a64_e0c62c1438a42621889aef2f44898dc9 store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:18:58,898 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/7a389d0c761b4ce3a21ac34710601714 is 50, key is test_row_0/B:col10/1731363538349/Put/seqid=0 2024-11-11T22:18:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:18:58,900 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111da515500b00149708f150b5717c30a64_e0c62c1438a42621889aef2f44898dc9, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:18:58,900 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111da515500b00149708f150b5717c30a64_e0c62c1438a42621889aef2f44898dc9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:18:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5f8f18c38c74bf6a6b841530670d8ee_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363538396/Put/seqid=0 2024-11-11T22:18:58,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742374_1550 (size=12104) 2024-11-11T22:18:58,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:58,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:18:58,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742375_1551 (size=4469) 2024-11-11T22:18:58,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742376_1552 (size=12154) 2024-11-11T22:18:58,996 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#A#compaction#471 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:58,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:18:58,996 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/fb973258006044a8b2997f0477b819f2 is 175, key is test_row_0/A:col10/1731363538349/Put/seqid=0 2024-11-11T22:18:59,000 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5f8f18c38c74bf6a6b841530670d8ee_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5f8f18c38c74bf6a6b841530670d8ee_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:18:59,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/5440611d10a94051a2f9e1d70c39a342, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:18:59,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/5440611d10a94051a2f9e1d70c39a342 is 175, key is test_row_0/A:col10/1731363538396/Put/seqid=0 2024-11-11T22:18:59,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742377_1553 (size=31058) 2024-11-11T22:18:59,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363599002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363599002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363599007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742378_1554 (size=30955) 2024-11-11T22:18:59,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363599029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363599029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363599109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363599119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363599119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363599320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363599329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363599329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,385 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/7a389d0c761b4ce3a21ac34710601714 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/7a389d0c761b4ce3a21ac34710601714 2024-11-11T22:18:59,402 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/B of e0c62c1438a42621889aef2f44898dc9 into 7a389d0c761b4ce3a21ac34710601714(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:59,402 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:18:59,402 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/B, priority=13, startTime=1731363538852; duration=0sec 2024-11-11T22:18:59,402 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:18:59,402 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:B 2024-11-11T22:18:59,402 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:18:59,408 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:18:59,408 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/C is initiating minor compaction (all files) 2024-11-11T22:18:59,408 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/C in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:18:59,408 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/86801e96e01046029baff540acd601ec, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/69b98701e25b4685a063e874f1a3d9f1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a84fb3043ac4416785dfede8ccc8dd3d] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=35.2 K 2024-11-11T22:18:59,409 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 86801e96e01046029baff540acd601ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731363536124 2024-11-11T22:18:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:18:59,410 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b98701e25b4685a063e874f1a3d9f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731363536174 2024-11-11T22:18:59,411 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a84fb3043ac4416785dfede8ccc8dd3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731363536826 2024-11-11T22:18:59,414 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/fb973258006044a8b2997f0477b819f2 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fb973258006044a8b2997f0477b819f2 2024-11-11T22:18:59,419 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/A of e0c62c1438a42621889aef2f44898dc9 into fb973258006044a8b2997f0477b819f2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:59,419 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:18:59,419 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/A, priority=13, startTime=1731363538849; duration=0sec 2024-11-11T22:18:59,419 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:59,419 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:A 2024-11-11T22:18:59,419 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:18:59,419 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/e114e0f67f9b4412a9f738d93d15362d is 50, key is test_row_0/C:col10/1731363538349/Put/seqid=0 2024-11-11T22:18:59,422 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/5440611d10a94051a2f9e1d70c39a342 2024-11-11T22:18:59,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742379_1555 (size=12104) 2024-11-11T22:18:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/af19c9ea95614212b51ffe6c53dcb759 is 50, key is test_row_0/B:col10/1731363538396/Put/seqid=0 2024-11-11T22:18:59,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742380_1556 (size=12001) 2024-11-11T22:18:59,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363599539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363599539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363599629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:18:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363599639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363599639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:18:59,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:18:59,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:18:59,841 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/e114e0f67f9b4412a9f738d93d15362d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/e114e0f67f9b4412a9f738d93d15362d 2024-11-11T22:18:59,843 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/af19c9ea95614212b51ffe6c53dcb759 2024-11-11T22:18:59,848 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/C of e0c62c1438a42621889aef2f44898dc9 into e114e0f67f9b4412a9f738d93d15362d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:18:59,848 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:18:59,848 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/C, priority=13, startTime=1731363538853; duration=0sec 2024-11-11T22:18:59,848 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:18:59,848 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:C 2024-11-11T22:18:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/f9827784797f4c7e873599b6ae3b41e9 is 50, key is test_row_0/C:col10/1731363538396/Put/seqid=0 2024-11-11T22:18:59,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742381_1557 (size=12001) 2024-11-11T22:19:00,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363600139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363600149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363600149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,264 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/f9827784797f4c7e873599b6ae3b41e9 2024-11-11T22:19:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/5440611d10a94051a2f9e1d70c39a342 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/5440611d10a94051a2f9e1d70c39a342 2024-11-11T22:19:00,278 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/5440611d10a94051a2f9e1d70c39a342, entries=150, sequenceid=81, filesize=30.2 K 2024-11-11T22:19:00,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/af19c9ea95614212b51ffe6c53dcb759 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/af19c9ea95614212b51ffe6c53dcb759 2024-11-11T22:19:00,294 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/af19c9ea95614212b51ffe6c53dcb759, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T22:19:00,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/f9827784797f4c7e873599b6ae3b41e9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f9827784797f4c7e873599b6ae3b41e9 2024-11-11T22:19:00,302 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f9827784797f4c7e873599b6ae3b41e9, entries=150, sequenceid=81, filesize=11.7 K 2024-11-11T22:19:00,305 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e0c62c1438a42621889aef2f44898dc9 in 1421ms, sequenceid=81, compaction requested=false 2024-11-11T22:19:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-11T22:19:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-11T22:19:00,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T22:19:00,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0360 sec 2024-11-11T22:19:00,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0410 sec 2024-11-11T22:19:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T22:19:00,419 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:00,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:19:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-11T22:19:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:19:00,421 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:19:00,422 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:19:00,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:19:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:19:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:00,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:19:00,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:00,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:00,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:00,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:00,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:00,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:00,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117e0385e0a20c4d64800aeee77fc497cf_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363539003/Put/seqid=0 2024-11-11T22:19:00,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742382_1558 (size=14594) 2024-11-11T22:19:00,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:19:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:00,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363600595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363600596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363600709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363600709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:19:00,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:00,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:00,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:00,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:19:00,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:19:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:00,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:00,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363600922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:00,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363600926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:00,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:00,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117e0385e0a20c4d64800aeee77fc497cf_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117e0385e0a20c4d64800aeee77fc497cf_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:00,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/3c0b7adea1f941e68f53b32d2605b14c, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:00,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/3c0b7adea1f941e68f53b32d2605b14c is 175, key is test_row_0/A:col10/1731363539003/Put/seqid=0 2024-11-11T22:19:00,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742383_1559 (size=39549) 2024-11-11T22:19:00,999 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/3c0b7adea1f941e68f53b32d2605b14c 2024-11-11T22:19:01,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/d15058e3f6934b4581797007b44c429a is 50, key is test_row_0/B:col10/1731363539003/Put/seqid=0 2024-11-11T22:19:01,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742384_1560 (size=12001) 2024-11-11T22:19:01,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/d15058e3f6934b4581797007b44c429a 2024-11-11T22:19:01,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/0459a4fe73c54070b1b140793c9c688f is 50, key is test_row_0/C:col10/1731363539003/Put/seqid=0 2024-11-11T22:19:01,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:19:01,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:01,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:19:01,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742385_1561 (size=12001) 2024-11-11T22:19:01,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363601149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,151 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363601159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,162 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_0' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363601169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,173 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:19:01,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:01,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363601229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363601239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:19:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:01,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/0459a4fe73c54070b1b140793c9c688f 2024-11-11T22:19:01,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/3c0b7adea1f941e68f53b32d2605b14c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3c0b7adea1f941e68f53b32d2605b14c 2024-11-11T22:19:01,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3c0b7adea1f941e68f53b32d2605b14c, entries=200, sequenceid=97, filesize=38.6 K 2024-11-11T22:19:01,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/d15058e3f6934b4581797007b44c429a as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d15058e3f6934b4581797007b44c429a 2024-11-11T22:19:01,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d15058e3f6934b4581797007b44c429a, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T22:19:01,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/0459a4fe73c54070b1b140793c9c688f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/0459a4fe73c54070b1b140793c9c688f 2024-11-11T22:19:01,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/0459a4fe73c54070b1b140793c9c688f, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T22:19:01,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e0c62c1438a42621889aef2f44898dc9 in 929ms, sequenceid=97, compaction requested=true 2024-11-11T22:19:01,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:01,481 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:01,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:19:01,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:01,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:19:01,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:01,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:19:01,481 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:01,482 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:01,482 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/B is initiating minor compaction (all files) 2024-11-11T22:19:01,482 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/B in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,482 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/7a389d0c761b4ce3a21ac34710601714, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/af19c9ea95614212b51ffe6c53dcb759, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d15058e3f6934b4581797007b44c429a] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=35.3 K 2024-11-11T22:19:01,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:01,488 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a389d0c761b4ce3a21ac34710601714, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731363536826 2024-11-11T22:19:01,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-11T22:19:01,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,496 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:19:01,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:01,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:01,496 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:01,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:01,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:01,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:01,496 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/A is initiating minor compaction (all files) 2024-11-11T22:19:01,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:01,496 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/A in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,496 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fb973258006044a8b2997f0477b819f2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/5440611d10a94051a2f9e1d70c39a342, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3c0b7adea1f941e68f53b32d2605b14c] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=99.2 K 2024-11-11T22:19:01,496 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:01,496 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fb973258006044a8b2997f0477b819f2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/5440611d10a94051a2f9e1d70c39a342, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3c0b7adea1f941e68f53b32d2605b14c] 2024-11-11T22:19:01,497 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting af19c9ea95614212b51ffe6c53dcb759, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731363538389 2024-11-11T22:19:01,497 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb973258006044a8b2997f0477b819f2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731363536826 2024-11-11T22:19:01,497 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d15058e3f6934b4581797007b44c429a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363538989 2024-11-11T22:19:01,497 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5440611d10a94051a2f9e1d70c39a342, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731363538389 2024-11-11T22:19:01,501 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c0b7adea1f941e68f53b32d2605b14c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363538989 2024-11-11T22:19:01,526 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#B#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:01,526 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/c4ac9185c5794551b0248e8d2491b502 is 50, key is test_row_0/B:col10/1731363539003/Put/seqid=0 2024-11-11T22:19:01,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e974a5da950347c1aa9b72ed49c03c31_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363540582/Put/seqid=0 2024-11-11T22:19:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:19:01,565 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:01,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742386_1562 (size=12207) 2024-11-11T22:19:01,585 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111197d5421edf174bfe89cc6bbd7df4d659_e0c62c1438a42621889aef2f44898dc9 store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:01,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742387_1563 (size=12154) 2024-11-11T22:19:01,587 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111197d5421edf174bfe89cc6bbd7df4d659_e0c62c1438a42621889aef2f44898dc9, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:01,587 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111197d5421edf174bfe89cc6bbd7df4d659_e0c62c1438a42621889aef2f44898dc9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:01,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742388_1564 (size=4469) 2024-11-11T22:19:01,614 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#A#compaction#482 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:01,615 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/0ec18b67c91945ea8a9ab3ab7363fe40 is 175, key is test_row_0/A:col10/1731363539003/Put/seqid=0 2024-11-11T22:19:01,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742389_1565 (size=31161) 2024-11-11T22:19:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:01,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363601775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363601779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363601893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363601893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:01,992 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/c4ac9185c5794551b0248e8d2491b502 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/c4ac9185c5794551b0248e8d2491b502 2024-11-11T22:19:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:02,001 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/B of e0c62c1438a42621889aef2f44898dc9 into c4ac9185c5794551b0248e8d2491b502(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:02,001 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:02,002 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/B, priority=13, startTime=1731363541481; duration=0sec 2024-11-11T22:19:02,002 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:02,002 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:B 2024-11-11T22:19:02,002 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:02,002 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:02,002 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/C is initiating minor compaction (all files) 2024-11-11T22:19:02,002 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/C in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:02,003 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/e114e0f67f9b4412a9f738d93d15362d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f9827784797f4c7e873599b6ae3b41e9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/0459a4fe73c54070b1b140793c9c688f] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=35.3 K 2024-11-11T22:19:02,003 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting e114e0f67f9b4412a9f738d93d15362d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731363536826 2024-11-11T22:19:02,003 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e974a5da950347c1aa9b72ed49c03c31_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e974a5da950347c1aa9b72ed49c03c31_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:02,003 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f9827784797f4c7e873599b6ae3b41e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731363538389 2024-11-11T22:19:02,004 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0459a4fe73c54070b1b140793c9c688f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363538989 2024-11-11T22:19:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/909b7ba156ad4b9fb8781c58eee40e67, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:02,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/909b7ba156ad4b9fb8781c58eee40e67 is 175, key is test_row_0/A:col10/1731363540582/Put/seqid=0 2024-11-11T22:19:02,031 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/0ec18b67c91945ea8a9ab3ab7363fe40 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/0ec18b67c91945ea8a9ab3ab7363fe40 2024-11-11T22:19:02,035 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/A of e0c62c1438a42621889aef2f44898dc9 into 0ec18b67c91945ea8a9ab3ab7363fe40(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:02,035 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:02,035 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/A, priority=13, startTime=1731363541481; duration=0sec 2024-11-11T22:19:02,035 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:02,036 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:A 2024-11-11T22:19:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742390_1566 (size=30955) 2024-11-11T22:19:02,058 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/909b7ba156ad4b9fb8781c58eee40e67 2024-11-11T22:19:02,070 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#C#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:02,070 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/61ad330168344bc89e69462dc128c484 is 50, key is test_row_0/C:col10/1731363539003/Put/seqid=0 2024-11-11T22:19:02,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/fdf80effb804419e8f4bde3d4c8d829c is 50, key is test_row_0/B:col10/1731363540582/Put/seqid=0 2024-11-11T22:19:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363602113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363602113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:02,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742391_1567 (size=12207) 2024-11-11T22:19:02,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742392_1568 (size=12001) 2024-11-11T22:19:02,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363602435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:02,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:02,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:02,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363602435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:02,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:02,540 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/fdf80effb804419e8f4bde3d4c8d829c 2024-11-11T22:19:02,554 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/61ad330168344bc89e69462dc128c484 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61ad330168344bc89e69462dc128c484 2024-11-11T22:19:02,567 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/C of e0c62c1438a42621889aef2f44898dc9 into 61ad330168344bc89e69462dc128c484(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:02,567 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:02,567 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/C, priority=13, startTime=1731363541481; duration=0sec 2024-11-11T22:19:02,567 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:02,567 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:C 2024-11-11T22:19:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:19:02,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/06889a60c73a46598eb88c4f2d064a50 is 50, key is test_row_0/C:col10/1731363540582/Put/seqid=0 2024-11-11T22:19:02,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742393_1569 (size=12001) 2024-11-11T22:19:02,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363602952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:02,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363602952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:02,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:03,040 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/06889a60c73a46598eb88c4f2d064a50 2024-11-11T22:19:03,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/909b7ba156ad4b9fb8781c58eee40e67 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/909b7ba156ad4b9fb8781c58eee40e67 2024-11-11T22:19:03,123 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/909b7ba156ad4b9fb8781c58eee40e67, entries=150, sequenceid=116, filesize=30.2 K 2024-11-11T22:19:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/fdf80effb804419e8f4bde3d4c8d829c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fdf80effb804419e8f4bde3d4c8d829c 2024-11-11T22:19:03,176 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fdf80effb804419e8f4bde3d4c8d829c, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T22:19:03,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/06889a60c73a46598eb88c4f2d064a50 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/06889a60c73a46598eb88c4f2d064a50 2024-11-11T22:19:03,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:03,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363603173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:03,190 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:03,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:03,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363603181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:03,195 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_0' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:03,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:03,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363603189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:03,199 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,207 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/06889a60c73a46598eb88c4f2d064a50, entries=150, sequenceid=116, filesize=11.7 K 2024-11-11T22:19:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,210 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e0c62c1438a42621889aef2f44898dc9 in 1714ms, sequenceid=116, compaction requested=false 2024-11-11T22:19:03,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:03,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:03,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-11T22:19:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-11T22:19:03,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-11T22:19:03,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7890 sec 2024-11-11T22:19:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.7980 sec 2024-11-11T22:19:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:03,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T22:19:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:03,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:03,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:03,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:03,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:03,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:03,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:04,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111129afd8c514542dc9aba19bc1e561187_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363543975/Put/seqid=0 2024-11-11T22:19:04,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742394_1570 (size=14694) 2024-11-11T22:19:04,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363604076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363604081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:04,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:04,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363604190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:04,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363604190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:04,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:04,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363604413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:04,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363604416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:04,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:04,437 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:04,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111129afd8c514542dc9aba19bc1e561187_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111129afd8c514542dc9aba19bc1e561187_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:04,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1ec291d46cf845c580b0ead58ac3cdd2, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:04,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1ec291d46cf845c580b0ead58ac3cdd2 is 175, key is test_row_0/A:col10/1731363543975/Put/seqid=0 2024-11-11T22:19:04,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742395_1571 (size=39649) 2024-11-11T22:19:04,473 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1ec291d46cf845c580b0ead58ac3cdd2 2024-11-11T22:19:04,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1c7a9751f0cb40479321d4d8f008ba2b is 50, key is test_row_0/B:col10/1731363543975/Put/seqid=0 2024-11-11T22:19:04,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742396_1572 (size=12101) 2024-11-11T22:19:04,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1c7a9751f0cb40479321d4d8f008ba2b 2024-11-11T22:19:04,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/2d81a3a5c57645e9814b007ea4a855e8 is 50, key is test_row_0/C:col10/1731363543975/Put/seqid=0 2024-11-11T22:19:04,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742397_1573 (size=12101) 2024-11-11T22:19:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T22:19:04,582 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:04,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:19:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-11T22:19:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:19:04,585 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:19:04,586 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:19:04,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:19:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:19:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363604729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363604732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:04,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T22:19:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:04,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:04,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T22:19:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:04,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:19:04,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/2d81a3a5c57645e9814b007ea4a855e8 2024-11-11T22:19:04,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1ec291d46cf845c580b0ead58ac3cdd2 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1ec291d46cf845c580b0ead58ac3cdd2 2024-11-11T22:19:04,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1ec291d46cf845c580b0ead58ac3cdd2, entries=200, sequenceid=134, filesize=38.7 K 2024-11-11T22:19:04,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1c7a9751f0cb40479321d4d8f008ba2b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1c7a9751f0cb40479321d4d8f008ba2b 2024-11-11T22:19:04,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1c7a9751f0cb40479321d4d8f008ba2b, entries=150, sequenceid=134, filesize=11.8 K 2024-11-11T22:19:04,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/2d81a3a5c57645e9814b007ea4a855e8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/2d81a3a5c57645e9814b007ea4a855e8 2024-11-11T22:19:04,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/2d81a3a5c57645e9814b007ea4a855e8, entries=150, sequenceid=134, filesize=11.8 K 2024-11-11T22:19:04,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e0c62c1438a42621889aef2f44898dc9 in 1000ms, sequenceid=134, compaction requested=true 2024-11-11T22:19:04,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:04,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:19:04,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:04,986 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:04,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:19:04,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:04,986 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:04,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:19:04,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:04,987 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:04,987 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/B is initiating minor compaction (all files) 2024-11-11T22:19:04,987 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/B in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:04,987 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/c4ac9185c5794551b0248e8d2491b502, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fdf80effb804419e8f4bde3d4c8d829c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1c7a9751f0cb40479321d4d8f008ba2b] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=35.5 K 2024-11-11T22:19:04,987 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:04,987 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/A is initiating minor compaction (all files) 2024-11-11T22:19:04,987 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/A in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:04,987 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/0ec18b67c91945ea8a9ab3ab7363fe40, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/909b7ba156ad4b9fb8781c58eee40e67, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1ec291d46cf845c580b0ead58ac3cdd2] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=99.4 K 2024-11-11T22:19:04,987 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:04,987 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/0ec18b67c91945ea8a9ab3ab7363fe40, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/909b7ba156ad4b9fb8781c58eee40e67, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1ec291d46cf845c580b0ead58ac3cdd2] 2024-11-11T22:19:04,988 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting c4ac9185c5794551b0248e8d2491b502, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363538989 2024-11-11T22:19:04,989 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec18b67c91945ea8a9ab3ab7363fe40, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363538989 2024-11-11T22:19:04,989 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting fdf80effb804419e8f4bde3d4c8d829c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731363540572 2024-11-11T22:19:04,989 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 909b7ba156ad4b9fb8781c58eee40e67, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731363540572 2024-11-11T22:19:04,989 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7a9751f0cb40479321d4d8f008ba2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363541772 2024-11-11T22:19:04,990 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ec291d46cf845c580b0ead58ac3cdd2, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363541772 2024-11-11T22:19:04,995 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:04,997 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#B#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:04,997 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/4bf5c1b4b70d4bd4ad742ec650be89ee is 50, key is test_row_0/B:col10/1731363543975/Put/seqid=0 2024-11-11T22:19:04,998 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c44a79579c904a34a43f66981fa4a795_e0c62c1438a42621889aef2f44898dc9 store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:05,000 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c44a79579c904a34a43f66981fa4a795_e0c62c1438a42621889aef2f44898dc9, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:05,000 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c44a79579c904a34a43f66981fa4a795_e0c62c1438a42621889aef2f44898dc9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:05,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742398_1574 (size=4469) 2024-11-11T22:19:05,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-11T22:19:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:05,045 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:19:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:05,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742399_1575 (size=12409) 2024-11-11T22:19:05,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bcfd5d5074de4115b6ad5eff64e3b8e2_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363544064/Put/seqid=0 2024-11-11T22:19:05,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742400_1576 (size=12304) 2024-11-11T22:19:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:19:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:05,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:05,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363605275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:05,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363605275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:05,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363605389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:05,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363605389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:05,425 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#A#compaction#489 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:05,426 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/828ff253807442b49c5251a1d8618220 is 175, key is test_row_0/A:col10/1731363543975/Put/seqid=0 2024-11-11T22:19:05,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742401_1577 (size=31363) 2024-11-11T22:19:05,461 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/4bf5c1b4b70d4bd4ad742ec650be89ee as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/4bf5c1b4b70d4bd4ad742ec650be89ee 2024-11-11T22:19:05,466 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/B of e0c62c1438a42621889aef2f44898dc9 into 4bf5c1b4b70d4bd4ad742ec650be89ee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:05,466 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:05,466 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/B, priority=13, startTime=1731363544986; duration=0sec 2024-11-11T22:19:05,466 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:05,466 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:B 2024-11-11T22:19:05,466 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:05,467 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:05,467 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/C is initiating minor compaction (all files) 2024-11-11T22:19:05,467 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/C in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:05,468 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61ad330168344bc89e69462dc128c484, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/06889a60c73a46598eb88c4f2d064a50, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/2d81a3a5c57645e9814b007ea4a855e8] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=35.5 K 2024-11-11T22:19:05,469 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 61ad330168344bc89e69462dc128c484, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731363538989 2024-11-11T22:19:05,469 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 06889a60c73a46598eb88c4f2d064a50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731363540572 2024-11-11T22:19:05,469 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d81a3a5c57645e9814b007ea4a855e8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363541772 2024-11-11T22:19:05,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:05,475 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bcfd5d5074de4115b6ad5eff64e3b8e2_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bcfd5d5074de4115b6ad5eff64e3b8e2_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:05,475 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:05,476 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/a9db1f70c9fd4c3fbd05c0f08d4fd08d is 50, key is test_row_0/C:col10/1731363543975/Put/seqid=0 2024-11-11T22:19:05,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/cffc9a2c56b041949f85a93fb1d174fc, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:05,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/cffc9a2c56b041949f85a93fb1d174fc is 175, key is test_row_0/A:col10/1731363544064/Put/seqid=0 2024-11-11T22:19:05,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742402_1578 (size=12409) 2024-11-11T22:19:05,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742403_1579 (size=31105) 2024-11-11T22:19:05,537 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/cffc9a2c56b041949f85a93fb1d174fc 2024-11-11T22:19:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/0392852d336a4d91b9b98ebd04ee0db5 is 50, key is test_row_0/B:col10/1731363544064/Put/seqid=0 2024-11-11T22:19:05,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742404_1580 (size=12151) 2024-11-11T22:19:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363605599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363605599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:19:05,843 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/828ff253807442b49c5251a1d8618220 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/828ff253807442b49c5251a1d8618220 2024-11-11T22:19:05,854 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/A of e0c62c1438a42621889aef2f44898dc9 into 828ff253807442b49c5251a1d8618220(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:05,854 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:05,854 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/A, priority=13, startTime=1731363544986; duration=0sec 2024-11-11T22:19:05,854 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:05,854 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:A 2024-11-11T22:19:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363605912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363605912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:05,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:05,930 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/a9db1f70c9fd4c3fbd05c0f08d4fd08d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a9db1f70c9fd4c3fbd05c0f08d4fd08d 2024-11-11T22:19:05,934 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/C of e0c62c1438a42621889aef2f44898dc9 into a9db1f70c9fd4c3fbd05c0f08d4fd08d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:05,934 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:05,934 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/C, priority=13, startTime=1731363544986; duration=0sec 2024-11-11T22:19:05,934 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:05,934 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:C 2024-11-11T22:19:05,988 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/0392852d336a4d91b9b98ebd04ee0db5 2024-11-11T22:19:06,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/bcfef6d7f0ec458c8108f7e1dee0ccb8 is 50, key is test_row_0/C:col10/1731363544064/Put/seqid=0 2024-11-11T22:19:06,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742405_1581 (size=12151) 2024-11-11T22:19:06,086 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/bcfef6d7f0ec458c8108f7e1dee0ccb8 2024-11-11T22:19:06,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/cffc9a2c56b041949f85a93fb1d174fc as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/cffc9a2c56b041949f85a93fb1d174fc 2024-11-11T22:19:06,094 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/cffc9a2c56b041949f85a93fb1d174fc, entries=150, sequenceid=151, filesize=30.4 K 2024-11-11T22:19:06,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/0392852d336a4d91b9b98ebd04ee0db5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/0392852d336a4d91b9b98ebd04ee0db5 2024-11-11T22:19:06,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/0392852d336a4d91b9b98ebd04ee0db5, entries=150, sequenceid=151, filesize=11.9 K 2024-11-11T22:19:06,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/bcfef6d7f0ec458c8108f7e1dee0ccb8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bcfef6d7f0ec458c8108f7e1dee0ccb8 2024-11-11T22:19:06,116 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bcfef6d7f0ec458c8108f7e1dee0ccb8, entries=150, sequenceid=151, filesize=11.9 K 2024-11-11T22:19:06,117 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e0c62c1438a42621889aef2f44898dc9 in 1072ms, sequenceid=151, compaction requested=false 2024-11-11T22:19:06,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:06,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:06,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-11T22:19:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-11T22:19:06,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T22:19:06,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-11-11T22:19:06,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5450 sec 2024-11-11T22:19:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:06,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T22:19:06,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:06,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:06,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:06,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:06,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:06,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:06,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cc95f630d7e84dbbaffe1273093c8b0f_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363545274/Put/seqid=0 2024-11-11T22:19:06,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742406_1582 (size=12304) 2024-11-11T22:19:06,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363606467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:06,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363606469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:06,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:06,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363606579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:06,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363606579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-11T22:19:06,730 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:06,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:19:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-11T22:19:06,732 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:19:06,733 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:19:06,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:19:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T22:19:06,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363606789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:06,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363606789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T22:19:06,848 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:06,851 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cc95f630d7e84dbbaffe1273093c8b0f_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc95f630d7e84dbbaffe1273093c8b0f_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:06,852 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/3f04ac3bdf8c4d7c94dcb0c452694d8d, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:06,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/3f04ac3bdf8c4d7c94dcb0c452694d8d is 175, key is test_row_0/A:col10/1731363545274/Put/seqid=0 2024-11-11T22:19:06,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742407_1583 (size=31105) 2024-11-11T22:19:06,858 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/3f04ac3bdf8c4d7c94dcb0c452694d8d 2024-11-11T22:19:06,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/b3c13e3d90ee4493b8bd70ec70843117 is 50, key is test_row_0/B:col10/1731363545274/Put/seqid=0 2024-11-11T22:19:06,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T22:19:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:06,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:06,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:06,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:06,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742408_1584 (size=12151) 2024-11-11T22:19:06,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/b3c13e3d90ee4493b8bd70ec70843117 2024-11-11T22:19:06,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/6341b6f0da4b46a2adc37554982d4467 is 50, key is test_row_0/C:col10/1731363545274/Put/seqid=0 2024-11-11T22:19:06,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742409_1585 (size=12151) 2024-11-11T22:19:07,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T22:19:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T22:19:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363607099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363607099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T22:19:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:07,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50808 deadline: 1731363607210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,211 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50812 deadline: 1731363607219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50890 deadline: 1731363607219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,220 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_2' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,221 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8d82b89dd27d,36911,1731363387997 for 'test_row_0' in e0c62c1438a42621889aef2f44898dc9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-11T22:19:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/6341b6f0da4b46a2adc37554982d4467 2024-11-11T22:19:07,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/3f04ac3bdf8c4d7c94dcb0c452694d8d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3f04ac3bdf8c4d7c94dcb0c452694d8d 2024-11-11T22:19:07,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3f04ac3bdf8c4d7c94dcb0c452694d8d, entries=150, sequenceid=170, filesize=30.4 K 2024-11-11T22:19:07,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/b3c13e3d90ee4493b8bd70ec70843117 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/b3c13e3d90ee4493b8bd70ec70843117 2024-11-11T22:19:07,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/b3c13e3d90ee4493b8bd70ec70843117, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T22:19:07,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/6341b6f0da4b46a2adc37554982d4467 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/6341b6f0da4b46a2adc37554982d4467 2024-11-11T22:19:07,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/6341b6f0da4b46a2adc37554982d4467, entries=150, sequenceid=170, filesize=11.9 K 2024-11-11T22:19:07,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e0c62c1438a42621889aef2f44898dc9 in 910ms, sequenceid=170, compaction requested=true 2024-11-11T22:19:07,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:07,330 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:07,332 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93573 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:07,332 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/A is initiating minor compaction (all files) 2024-11-11T22:19:07,332 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/A in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,332 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/828ff253807442b49c5251a1d8618220, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/cffc9a2c56b041949f85a93fb1d174fc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3f04ac3bdf8c4d7c94dcb0c452694d8d] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=91.4 K 2024-11-11T22:19:07,332 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,332 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/828ff253807442b49c5251a1d8618220, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/cffc9a2c56b041949f85a93fb1d174fc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3f04ac3bdf8c4d7c94dcb0c452694d8d] 2024-11-11T22:19:07,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:19:07,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:07,332 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 828ff253807442b49c5251a1d8618220, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363541772 2024-11-11T22:19:07,332 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting cffc9a2c56b041949f85a93fb1d174fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731363544064 2024-11-11T22:19:07,333 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f04ac3bdf8c4d7c94dcb0c452694d8d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363545267 2024-11-11T22:19:07,333 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:19:07,334 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:07,334 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/B is initiating minor compaction (all files) 2024-11-11T22:19:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:07,334 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/B in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:19:07,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:07,335 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/4bf5c1b4b70d4bd4ad742ec650be89ee, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/0392852d336a4d91b9b98ebd04ee0db5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/b3c13e3d90ee4493b8bd70ec70843117] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=35.9 K 2024-11-11T22:19:07,335 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf5c1b4b70d4bd4ad742ec650be89ee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363541772 2024-11-11T22:19:07,335 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 0392852d336a4d91b9b98ebd04ee0db5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731363544064 2024-11-11T22:19:07,335 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting b3c13e3d90ee4493b8bd70ec70843117, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363545267 2024-11-11T22:19:07,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-11T22:19:07,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,347 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:19:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:07,353 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T22:19:07,360 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#B#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:07,361 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/91523b0b290b460ba3fbd5c7d750a503 is 50, key is test_row_0/B:col10/1731363545274/Put/seqid=0 2024-11-11T22:19:07,378 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111fa92eb60f3de47c38d1fcd43e7849363_e0c62c1438a42621889aef2f44898dc9 store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:07,385 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111fa92eb60f3de47c38d1fcd43e7849363_e0c62c1438a42621889aef2f44898dc9, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:07,385 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fa92eb60f3de47c38d1fcd43e7849363_e0c62c1438a42621889aef2f44898dc9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:07,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a75a8f94817e432cbfb16657f09e0ca5_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363546460/Put/seqid=0 2024-11-11T22:19:07,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742410_1586 (size=12561) 2024-11-11T22:19:07,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742412_1588 (size=12304) 2024-11-11T22:19:07,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742411_1587 (size=4469) 2024-11-11T22:19:07,445 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/91523b0b290b460ba3fbd5c7d750a503 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/91523b0b290b460ba3fbd5c7d750a503 2024-11-11T22:19:07,462 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/B of e0c62c1438a42621889aef2f44898dc9 into 91523b0b290b460ba3fbd5c7d750a503(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:07,462 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:07,462 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/B, priority=13, startTime=1731363547333; duration=0sec 2024-11-11T22:19:07,462 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:07,462 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:B 2024-11-11T22:19:07,462 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:07,463 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:07,463 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/C is initiating minor compaction (all files) 2024-11-11T22:19:07,463 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/C in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:07,463 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a9db1f70c9fd4c3fbd05c0f08d4fd08d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bcfef6d7f0ec458c8108f7e1dee0ccb8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/6341b6f0da4b46a2adc37554982d4467] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=35.9 K 2024-11-11T22:19:07,465 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a9db1f70c9fd4c3fbd05c0f08d4fd08d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731363541772 2024-11-11T22:19:07,465 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bcfef6d7f0ec458c8108f7e1dee0ccb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731363544064 2024-11-11T22:19:07,466 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 6341b6f0da4b46a2adc37554982d4467, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363545267 2024-11-11T22:19:07,477 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:07,478 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/81dbf73b78224da4a111034429c31fe6 is 50, key is test_row_0/C:col10/1731363545274/Put/seqid=0 2024-11-11T22:19:07,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742413_1589 (size=12561) 2024-11-11T22:19:07,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:07,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363607666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363607673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363607780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363607789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:07,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:07,836 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#A#compaction#498 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:07,837 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/ff47d5407ec44fa8a2b703007c5bb0a5 is 175, key is test_row_0/A:col10/1731363545274/Put/seqid=0 2024-11-11T22:19:07,838 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a75a8f94817e432cbfb16657f09e0ca5_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a75a8f94817e432cbfb16657f09e0ca5_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/61fa061dcfbf47a793142cb82f28c327, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:07,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/61fa061dcfbf47a793142cb82f28c327 is 175, key is test_row_0/A:col10/1731363546460/Put/seqid=0 2024-11-11T22:19:07,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742414_1590 (size=31515) 2024-11-11T22:19:07,861 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/ff47d5407ec44fa8a2b703007c5bb0a5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/ff47d5407ec44fa8a2b703007c5bb0a5 2024-11-11T22:19:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T22:19:07,870 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/A of e0c62c1438a42621889aef2f44898dc9 into ff47d5407ec44fa8a2b703007c5bb0a5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:07,870 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:07,870 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/A, priority=13, startTime=1731363547330; duration=0sec 2024-11-11T22:19:07,870 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:07,870 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:A 2024-11-11T22:19:07,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742415_1591 (size=31105) 2024-11-11T22:19:07,873 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/61fa061dcfbf47a793142cb82f28c327 2024-11-11T22:19:07,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/72d8b13d7dd148cb943bb68c809c2a6f is 50, key is test_row_0/B:col10/1731363546460/Put/seqid=0 2024-11-11T22:19:07,932 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/81dbf73b78224da4a111034429c31fe6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/81dbf73b78224da4a111034429c31fe6 2024-11-11T22:19:07,936 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/C of e0c62c1438a42621889aef2f44898dc9 into 81dbf73b78224da4a111034429c31fe6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:07,937 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:07,937 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/C, priority=13, startTime=1731363547334; duration=0sec 2024-11-11T22:19:07,937 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:07,937 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:C 2024-11-11T22:19:07,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742416_1592 (size=12151) 2024-11-11T22:19:07,956 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/72d8b13d7dd148cb943bb68c809c2a6f 2024-11-11T22:19:07,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/473da8663fd9450cabaed5232f704ce5 is 50, key is test_row_0/C:col10/1731363546460/Put/seqid=0 2024-11-11T22:19:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363607989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:08,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363608001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:08,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742417_1593 (size=12151) 2024-11-11T22:19:08,032 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/473da8663fd9450cabaed5232f704ce5 2024-11-11T22:19:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/61fa061dcfbf47a793142cb82f28c327 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/61fa061dcfbf47a793142cb82f28c327 2024-11-11T22:19:08,047 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/61fa061dcfbf47a793142cb82f28c327, entries=150, sequenceid=187, filesize=30.4 K 2024-11-11T22:19:08,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/72d8b13d7dd148cb943bb68c809c2a6f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/72d8b13d7dd148cb943bb68c809c2a6f 2024-11-11T22:19:08,052 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/72d8b13d7dd148cb943bb68c809c2a6f, entries=150, sequenceid=187, filesize=11.9 K 2024-11-11T22:19:08,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/473da8663fd9450cabaed5232f704ce5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/473da8663fd9450cabaed5232f704ce5 2024-11-11T22:19:08,065 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/473da8663fd9450cabaed5232f704ce5, entries=150, sequenceid=187, filesize=11.9 K 2024-11-11T22:19:08,067 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e0c62c1438a42621889aef2f44898dc9 in 719ms, sequenceid=187, compaction requested=false 2024-11-11T22:19:08,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:08,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:08,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-11T22:19:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-11T22:19:08,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T22:19:08,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3350 sec 2024-11-11T22:19:08,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.3430 sec 2024-11-11T22:19:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:08,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T22:19:08,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:08,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:08,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115135d078b1f941db8f7a9787069dc6e9_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363547655/Put/seqid=0 2024-11-11T22:19:08,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742418_1594 (size=12304) 2024-11-11T22:19:08,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:08,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115135d078b1f941db8f7a9787069dc6e9_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115135d078b1f941db8f7a9787069dc6e9_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:08,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/e6909ae59fa340028054044d329932e7, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:08,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/e6909ae59fa340028054044d329932e7 is 175, key is test_row_0/A:col10/1731363547655/Put/seqid=0 2024-11-11T22:19:08,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363608342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:08,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363608348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:08,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742419_1595 (size=31105) 2024-11-11T22:19:08,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363608459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363608459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:08,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:08,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363608669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363608670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:08,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:08,754 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/e6909ae59fa340028054044d329932e7 2024-11-11T22:19:08,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/8c14f18db42c438db68c93ea6c8bede3 is 50, key is test_row_0/B:col10/1731363547655/Put/seqid=0 2024-11-11T22:19:08,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742420_1596 (size=12151) 2024-11-11T22:19:08,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/8c14f18db42c438db68c93ea6c8bede3 2024-11-11T22:19:08,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/ff8dcdc2191f4ce3876d6a5d31f1f253 is 50, key is test_row_0/C:col10/1731363547655/Put/seqid=0 2024-11-11T22:19:08,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742421_1597 (size=12151) 2024-11-11T22:19:08,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/ff8dcdc2191f4ce3876d6a5d31f1f253 2024-11-11T22:19:08,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/e6909ae59fa340028054044d329932e7 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/e6909ae59fa340028054044d329932e7 2024-11-11T22:19:08,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/e6909ae59fa340028054044d329932e7, entries=150, sequenceid=206, filesize=30.4 K 2024-11-11T22:19:08,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/8c14f18db42c438db68c93ea6c8bede3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/8c14f18db42c438db68c93ea6c8bede3 2024-11-11T22:19:08,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/8c14f18db42c438db68c93ea6c8bede3, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T22:19:08,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/ff8dcdc2191f4ce3876d6a5d31f1f253 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/ff8dcdc2191f4ce3876d6a5d31f1f253 2024-11-11T22:19:08,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/ff8dcdc2191f4ce3876d6a5d31f1f253, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T22:19:08,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e0c62c1438a42621889aef2f44898dc9 in 512ms, sequenceid=206, compaction requested=true 2024-11-11T22:19:08,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:08,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:19:08,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:08,814 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:08,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:19:08,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:08,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:19:08,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:08,814 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:08,814 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:08,814 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/B is initiating minor compaction (all files) 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/A is initiating minor compaction (all files) 2024-11-11T22:19:08,815 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/B in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:08,815 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/A in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:08,815 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/ff47d5407ec44fa8a2b703007c5bb0a5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/61fa061dcfbf47a793142cb82f28c327, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/e6909ae59fa340028054044d329932e7] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=91.5 K 2024-11-11T22:19:08,815 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/91523b0b290b460ba3fbd5c7d750a503, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/72d8b13d7dd148cb943bb68c809c2a6f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/8c14f18db42c438db68c93ea6c8bede3] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=36.0 K 2024-11-11T22:19:08,815 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/ff47d5407ec44fa8a2b703007c5bb0a5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/61fa061dcfbf47a793142cb82f28c327, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/e6909ae59fa340028054044d329932e7] 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff47d5407ec44fa8a2b703007c5bb0a5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363545267 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 91523b0b290b460ba3fbd5c7d750a503, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363545267 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61fa061dcfbf47a793142cb82f28c327, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731363546422 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d8b13d7dd148cb943bb68c809c2a6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731363546422 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6909ae59fa340028054044d329932e7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731363547655 2024-11-11T22:19:08,815 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c14f18db42c438db68c93ea6c8bede3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731363547655 2024-11-11T22:19:08,819 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:08,820 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411117c534858909b40b793c8437003aacc3d_e0c62c1438a42621889aef2f44898dc9 store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:08,820 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:08,821 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/11c9a2cce1c044ce8588bc69a4d49dca is 50, key is test_row_0/B:col10/1731363547655/Put/seqid=0 2024-11-11T22:19:08,822 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411117c534858909b40b793c8437003aacc3d_e0c62c1438a42621889aef2f44898dc9, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:08,822 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117c534858909b40b793c8437003aacc3d_e0c62c1438a42621889aef2f44898dc9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:08,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742422_1598 (size=12663) 2024-11-11T22:19:08,857 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/11c9a2cce1c044ce8588bc69a4d49dca as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/11c9a2cce1c044ce8588bc69a4d49dca 2024-11-11T22:19:08,862 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/B of e0c62c1438a42621889aef2f44898dc9 into 11c9a2cce1c044ce8588bc69a4d49dca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:08,862 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:08,862 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/B, priority=13, startTime=1731363548814; duration=0sec 2024-11-11T22:19:08,862 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:08,862 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:B 2024-11-11T22:19:08,863 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:08,864 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:08,864 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/C is initiating minor compaction (all files) 2024-11-11T22:19:08,864 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/C in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:08,864 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/81dbf73b78224da4a111034429c31fe6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/473da8663fd9450cabaed5232f704ce5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/ff8dcdc2191f4ce3876d6a5d31f1f253] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=36.0 K 2024-11-11T22:19:08,865 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 81dbf73b78224da4a111034429c31fe6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731363545267 2024-11-11T22:19:08,865 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 473da8663fd9450cabaed5232f704ce5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731363546422 2024-11-11T22:19:08,866 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8dcdc2191f4ce3876d6a5d31f1f253, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731363547655 2024-11-11T22:19:08,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742423_1599 (size=4469) 2024-11-11T22:19:08,873 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:08,873 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/37402f3d14d349b08cf2aa7099441ebd is 50, key is test_row_0/C:col10/1731363547655/Put/seqid=0 2024-11-11T22:19:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-11T22:19:08,880 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:08,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:19:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-11T22:19:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T22:19:08,889 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:19:08,892 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:19:08,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:19:08,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742424_1600 (size=12663) 2024-11-11T22:19:08,903 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/37402f3d14d349b08cf2aa7099441ebd as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/37402f3d14d349b08cf2aa7099441ebd 2024-11-11T22:19:08,909 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/C of e0c62c1438a42621889aef2f44898dc9 into 37402f3d14d349b08cf2aa7099441ebd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:08,909 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:08,909 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/C, priority=13, startTime=1731363548814; duration=0sec 2024-11-11T22:19:08,909 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:08,909 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:C 2024-11-11T22:19:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:08,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T22:19:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T22:19:08,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c0ff349e8dee4953a56c52e5f917d385_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363548346/Put/seqid=0 2024-11-11T22:19:09,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742425_1601 (size=17284) 2024-11-11T22:19:09,028 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:09,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c0ff349e8dee4953a56c52e5f917d385_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0ff349e8dee4953a56c52e5f917d385_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:09,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363609026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363609029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,036 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/b67dd23c2c7d452aad77c4185a93670b, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/b67dd23c2c7d452aad77c4185a93670b is 175, key is test_row_0/A:col10/1731363548346/Put/seqid=0 2024-11-11T22:19:09,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T22:19:09,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:09,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:09,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:09,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742426_1602 (size=48389) 2024-11-11T22:19:09,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/b67dd23c2c7d452aad77c4185a93670b 2024-11-11T22:19:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/d524ac656b7a47e59ec18bf1839ce0fe is 50, key is test_row_0/B:col10/1731363548346/Put/seqid=0 2024-11-11T22:19:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742427_1603 (size=12151) 2024-11-11T22:19:09,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/d524ac656b7a47e59ec18bf1839ce0fe 2024-11-11T22:19:09,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/bf00120e010e47b7993f61ee34c54e0c is 50, key is test_row_0/C:col10/1731363548346/Put/seqid=0 2024-11-11T22:19:09,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742428_1604 (size=12151) 2024-11-11T22:19:09,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363609139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363609139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T22:19:09,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T22:19:09,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:09,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:09,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:09,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,269 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#A#compaction#507 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:09,269 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/dadd7a862d1f4a5784f5bd664e302497 is 175, key is test_row_0/A:col10/1731363547655/Put/seqid=0 2024-11-11T22:19:09,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742429_1605 (size=31617) 2024-11-11T22:19:09,297 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/dadd7a862d1f4a5784f5bd664e302497 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dadd7a862d1f4a5784f5bd664e302497 2024-11-11T22:19:09,301 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/A of e0c62c1438a42621889aef2f44898dc9 into dadd7a862d1f4a5784f5bd664e302497(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:09,302 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:09,302 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/A, priority=13, startTime=1731363548814; duration=0sec 2024-11-11T22:19:09,302 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:09,302 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:A 2024-11-11T22:19:09,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363609349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363609350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T22:19:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:09,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:09,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/bf00120e010e47b7993f61ee34c54e0c 2024-11-11T22:19:09,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/b67dd23c2c7d452aad77c4185a93670b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/b67dd23c2c7d452aad77c4185a93670b 2024-11-11T22:19:09,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/b67dd23c2c7d452aad77c4185a93670b, entries=250, sequenceid=226, filesize=47.3 K 2024-11-11T22:19:09,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/d524ac656b7a47e59ec18bf1839ce0fe as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d524ac656b7a47e59ec18bf1839ce0fe 2024-11-11T22:19:09,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d524ac656b7a47e59ec18bf1839ce0fe, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T22:19:09,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/bf00120e010e47b7993f61ee34c54e0c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bf00120e010e47b7993f61ee34c54e0c 2024-11-11T22:19:09,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bf00120e010e47b7993f61ee34c54e0c, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T22:19:09,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e0c62c1438a42621889aef2f44898dc9 in 524ms, sequenceid=226, compaction requested=false 2024-11-11T22:19:09,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T22:19:09,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T22:19:09,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:09,520 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T22:19:09,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:09,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:09,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:09,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:09,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:09,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ff3bb5e47b624fdaa07ec827b99074aa_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363549002/Put/seqid=0 2024-11-11T22:19:09,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742430_1606 (size=12304) 2024-11-11T22:19:09,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:09,565 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ff3bb5e47b624fdaa07ec827b99074aa_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ff3bb5e47b624fdaa07ec827b99074aa_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:09,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/9023b4b6e7614bb3a56779dea7df2b74, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:09,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/9023b4b6e7614bb3a56779dea7df2b74 is 175, key is test_row_0/A:col10/1731363549002/Put/seqid=0 2024-11-11T22:19:09,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742431_1607 (size=31105) 2024-11-11T22:19:09,605 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/9023b4b6e7614bb3a56779dea7df2b74 2024-11-11T22:19:09,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1d5339b1bd814bb593ac2960044bde8e is 50, key is test_row_0/B:col10/1731363549002/Put/seqid=0 2024-11-11T22:19:09,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742432_1608 (size=12151) 2024-11-11T22:19:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:09,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:09,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363609708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363609715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363609820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,823 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:09,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363609829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T22:19:10,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363610030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:10,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363610039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:10,059 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1d5339b1bd814bb593ac2960044bde8e 2024-11-11T22:19:10,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/f5bd0ed55623432086332cd28e012fa6 is 50, key is test_row_0/C:col10/1731363549002/Put/seqid=0 2024-11-11T22:19:10,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742433_1609 (size=12151) 2024-11-11T22:19:10,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363610339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:10,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:10,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363610349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:10,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:10,475 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/f5bd0ed55623432086332cd28e012fa6 2024-11-11T22:19:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/9023b4b6e7614bb3a56779dea7df2b74 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/9023b4b6e7614bb3a56779dea7df2b74 2024-11-11T22:19:10,482 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/9023b4b6e7614bb3a56779dea7df2b74, entries=150, sequenceid=241, filesize=30.4 K 2024-11-11T22:19:10,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1d5339b1bd814bb593ac2960044bde8e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1d5339b1bd814bb593ac2960044bde8e 2024-11-11T22:19:10,486 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1d5339b1bd814bb593ac2960044bde8e, entries=150, sequenceid=241, filesize=11.9 K 2024-11-11T22:19:10,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/f5bd0ed55623432086332cd28e012fa6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f5bd0ed55623432086332cd28e012fa6 2024-11-11T22:19:10,490 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f5bd0ed55623432086332cd28e012fa6, entries=150, sequenceid=241, filesize=11.9 K 2024-11-11T22:19:10,491 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e0c62c1438a42621889aef2f44898dc9 in 971ms, sequenceid=241, compaction requested=true 2024-11-11T22:19:10,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:10,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:10,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-11T22:19:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-11T22:19:10,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-11T22:19:10,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-11-11T22:19:10,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.6160 sec 2024-11-11T22:19:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:10,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T22:19:10,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:10,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:10,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:10,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:10,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:10,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:10,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111332d1a7277b8458ebb2d2b8d1004eb3f_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363549693/Put/seqid=0 2024-11-11T22:19:10,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742434_1610 (size=14894) 2024-11-11T22:19:10,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:10,876 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111332d1a7277b8458ebb2d2b8d1004eb3f_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111332d1a7277b8458ebb2d2b8d1004eb3f_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:10,876 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/21f68c9999424361a53a43b7699fc614, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:10,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/21f68c9999424361a53a43b7699fc614 is 175, key is test_row_0/A:col10/1731363549693/Put/seqid=0 2024-11-11T22:19:10,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742435_1611 (size=39849) 2024-11-11T22:19:10,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363610914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:10,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363610918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:10,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363611029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363611029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T22:19:11,042 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:11,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:19:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-11T22:19:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T22:19:11,046 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:19:11,046 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:19:11,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:19:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T22:19:11,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T22:19:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:11,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363611239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363611239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,320 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/21f68c9999424361a53a43b7699fc614 2024-11-11T22:19:11,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/82d0970df9b144c89455cc6f66833c48 is 50, key is test_row_0/B:col10/1731363549693/Put/seqid=0 2024-11-11T22:19:11,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742436_1612 (size=12201) 2024-11-11T22:19:11,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/82d0970df9b144c89455cc6f66833c48 2024-11-11T22:19:11,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/61458fdae04d46d5924ab47bec6973e5 is 50, key is test_row_0/C:col10/1731363549693/Put/seqid=0 2024-11-11T22:19:11,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T22:19:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:11,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742437_1613 (size=12201) 2024-11-11T22:19:11,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/61458fdae04d46d5924ab47bec6973e5 2024-11-11T22:19:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T22:19:11,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/21f68c9999424361a53a43b7699fc614 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/21f68c9999424361a53a43b7699fc614 2024-11-11T22:19:11,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/21f68c9999424361a53a43b7699fc614, entries=200, sequenceid=260, filesize=38.9 K 2024-11-11T22:19:11,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/82d0970df9b144c89455cc6f66833c48 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/82d0970df9b144c89455cc6f66833c48 2024-11-11T22:19:11,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/82d0970df9b144c89455cc6f66833c48, entries=150, sequenceid=260, filesize=11.9 K 2024-11-11T22:19:11,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/61458fdae04d46d5924ab47bec6973e5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61458fdae04d46d5924ab47bec6973e5 2024-11-11T22:19:11,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61458fdae04d46d5924ab47bec6973e5, entries=150, sequenceid=260, filesize=11.9 K 2024-11-11T22:19:11,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e0c62c1438a42621889aef2f44898dc9 in 522ms, sequenceid=260, compaction requested=true 2024-11-11T22:19:11,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:19:11,379 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:19:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:11,379 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:19:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:19:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:19:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150960 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/A is initiating minor compaction (all files) 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/B is initiating minor compaction (all files) 2024-11-11T22:19:11,382 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/A in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,382 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/B in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,382 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/11c9a2cce1c044ce8588bc69a4d49dca, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d524ac656b7a47e59ec18bf1839ce0fe, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1d5339b1bd814bb593ac2960044bde8e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/82d0970df9b144c89455cc6f66833c48] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=48.0 K 2024-11-11T22:19:11,382 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dadd7a862d1f4a5784f5bd664e302497, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/b67dd23c2c7d452aad77c4185a93670b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/9023b4b6e7614bb3a56779dea7df2b74, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/21f68c9999424361a53a43b7699fc614] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=147.4 K 2024-11-11T22:19:11,382 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dadd7a862d1f4a5784f5bd664e302497, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/b67dd23c2c7d452aad77c4185a93670b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/9023b4b6e7614bb3a56779dea7df2b74, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/21f68c9999424361a53a43b7699fc614] 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 11c9a2cce1c044ce8588bc69a4d49dca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731363547655 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting dadd7a862d1f4a5784f5bd664e302497, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731363547655 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d524ac656b7a47e59ec18bf1839ce0fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363548346 2024-11-11T22:19:11,382 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting b67dd23c2c7d452aad77c4185a93670b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363548340 2024-11-11T22:19:11,383 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d5339b1bd814bb593ac2960044bde8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731363549002 2024-11-11T22:19:11,384 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 82d0970df9b144c89455cc6f66833c48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363549693 2024-11-11T22:19:11,384 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9023b4b6e7614bb3a56779dea7df2b74, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731363549002 2024-11-11T22:19:11,385 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21f68c9999424361a53a43b7699fc614, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363549693 2024-11-11T22:19:11,402 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#B#compaction#519 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:11,404 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/ccf0cf34a4004a719e729f43d40db882 is 50, key is test_row_0/B:col10/1731363549693/Put/seqid=0 2024-11-11T22:19:11,409 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:11,417 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111336dc1bfccf74832ba6b2fbf9e9e2bfb_e0c62c1438a42621889aef2f44898dc9 store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:11,421 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111336dc1bfccf74832ba6b2fbf9e9e2bfb_e0c62c1438a42621889aef2f44898dc9, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:11,421 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111336dc1bfccf74832ba6b2fbf9e9e2bfb_e0c62c1438a42621889aef2f44898dc9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:11,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742438_1614 (size=12849) 2024-11-11T22:19:11,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742439_1615 (size=4469) 2024-11-11T22:19:11,441 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#A#compaction#520 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:11,442 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/2a563a4dac7244bca1353efc1c8f2e09 is 175, key is test_row_0/A:col10/1731363549693/Put/seqid=0 2024-11-11T22:19:11,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742440_1616 (size=31803) 2024-11-11T22:19:11,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T22:19:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,507 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T22:19:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:11,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:11,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5b22c3c2695429cbfdf4152ff29b233_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363550914/Put/seqid=0 2024-11-11T22:19:11,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742441_1617 (size=12454) 2024-11-11T22:19:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:11,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:11,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363611623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363611629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T22:19:11,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363611739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363611739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,837 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/ccf0cf34a4004a719e729f43d40db882 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/ccf0cf34a4004a719e729f43d40db882 2024-11-11T22:19:11,842 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/B of e0c62c1438a42621889aef2f44898dc9 into ccf0cf34a4004a719e729f43d40db882(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:11,842 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:11,842 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/B, priority=12, startTime=1731363551379; duration=0sec 2024-11-11T22:19:11,842 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:11,842 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:B 2024-11-11T22:19:11,842 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T22:19:11,843 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49166 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T22:19:11,843 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/C is initiating minor compaction (all files) 2024-11-11T22:19:11,843 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/C in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:11,843 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/37402f3d14d349b08cf2aa7099441ebd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bf00120e010e47b7993f61ee34c54e0c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f5bd0ed55623432086332cd28e012fa6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61458fdae04d46d5924ab47bec6973e5] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=48.0 K 2024-11-11T22:19:11,843 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 37402f3d14d349b08cf2aa7099441ebd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731363547655 2024-11-11T22:19:11,843 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting bf00120e010e47b7993f61ee34c54e0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731363548346 2024-11-11T22:19:11,844 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f5bd0ed55623432086332cd28e012fa6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731363549002 2024-11-11T22:19:11,844 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 61458fdae04d46d5924ab47bec6973e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363549693 2024-11-11T22:19:11,851 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/2a563a4dac7244bca1353efc1c8f2e09 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/2a563a4dac7244bca1353efc1c8f2e09 2024-11-11T22:19:11,854 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#C#compaction#522 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:11,854 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/58472634186f4f8b8a6cba1123550ec8 is 50, key is test_row_0/C:col10/1731363549693/Put/seqid=0 2024-11-11T22:19:11,857 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/A of e0c62c1438a42621889aef2f44898dc9 into 2a563a4dac7244bca1353efc1c8f2e09(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:11,857 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:11,857 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/A, priority=12, startTime=1731363551379; duration=0sec 2024-11-11T22:19:11,857 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:11,857 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:A 2024-11-11T22:19:11,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742442_1618 (size=12849) 2024-11-11T22:19:11,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:11,931 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5b22c3c2695429cbfdf4152ff29b233_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5b22c3c2695429cbfdf4152ff29b233_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:11,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/fcdc4e1d90eb4701bac24ee2cf5c4138, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:11,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/fcdc4e1d90eb4701bac24ee2cf5c4138 is 175, key is test_row_0/A:col10/1731363550914/Put/seqid=0 2024-11-11T22:19:11,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742443_1619 (size=31255) 2024-11-11T22:19:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363611949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363611949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:11,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T22:19:12,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363612260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:12,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363612260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:12,284 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/58472634186f4f8b8a6cba1123550ec8 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/58472634186f4f8b8a6cba1123550ec8 2024-11-11T22:19:12,297 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/C of e0c62c1438a42621889aef2f44898dc9 into 58472634186f4f8b8a6cba1123550ec8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:12,297 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:12,297 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/C, priority=12, startTime=1731363551380; duration=0sec 2024-11-11T22:19:12,297 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:12,297 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:C 2024-11-11T22:19:12,339 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/fcdc4e1d90eb4701bac24ee2cf5c4138 2024-11-11T22:19:12,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/babbd5c20f5d40a0bdcaac83e8e8bca9 is 50, key is test_row_0/B:col10/1731363550914/Put/seqid=0 2024-11-11T22:19:12,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742444_1620 (size=12301) 2024-11-11T22:19:12,364 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/babbd5c20f5d40a0bdcaac83e8e8bca9 2024-11-11T22:19:12,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/22af7cab1cfc4e8daee27793d7edb232 is 50, key is test_row_0/C:col10/1731363550914/Put/seqid=0 2024-11-11T22:19:12,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742445_1621 (size=12301) 2024-11-11T22:19:12,404 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/22af7cab1cfc4e8daee27793d7edb232 2024-11-11T22:19:12,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/fcdc4e1d90eb4701bac24ee2cf5c4138 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fcdc4e1d90eb4701bac24ee2cf5c4138 2024-11-11T22:19:12,429 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fcdc4e1d90eb4701bac24ee2cf5c4138, entries=150, sequenceid=273, filesize=30.5 K 2024-11-11T22:19:12,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/babbd5c20f5d40a0bdcaac83e8e8bca9 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/babbd5c20f5d40a0bdcaac83e8e8bca9 2024-11-11T22:19:12,441 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/babbd5c20f5d40a0bdcaac83e8e8bca9, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T22:19:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/22af7cab1cfc4e8daee27793d7edb232 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/22af7cab1cfc4e8daee27793d7edb232 2024-11-11T22:19:12,453 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/22af7cab1cfc4e8daee27793d7edb232, entries=150, sequenceid=273, filesize=12.0 K 2024-11-11T22:19:12,454 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e0c62c1438a42621889aef2f44898dc9 in 947ms, sequenceid=273, compaction requested=false 2024-11-11T22:19:12,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:12,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:12,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-11T22:19:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-11T22:19:12,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-11T22:19:12,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4110 sec 2024-11-11T22:19:12,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4160 sec 2024-11-11T22:19:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:12,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T22:19:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:12,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112dfc4b2b8c634248a8bc0e8e027b59d8_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363551614/Put/seqid=0 2024-11-11T22:19:12,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742446_1622 (size=14994) 2024-11-11T22:19:12,795 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:12,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112dfc4b2b8c634248a8bc0e8e027b59d8_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112dfc4b2b8c634248a8bc0e8e027b59d8_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:12,799 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/aac4d3222c0e47c8b7330b7d99518063, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:12,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/aac4d3222c0e47c8b7330b7d99518063 is 175, key is test_row_0/A:col10/1731363551614/Put/seqid=0 2024-11-11T22:19:12,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363612797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:12,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742447_1623 (size=39949) 2024-11-11T22:19:12,803 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/aac4d3222c0e47c8b7330b7d99518063 2024-11-11T22:19:12,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363612802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:12,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/61289452fd154e7ead712eedf3fec970 is 50, key is test_row_0/B:col10/1731363551614/Put/seqid=0 2024-11-11T22:19:12,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742448_1624 (size=12301) 2024-11-11T22:19:12,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363612909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:12,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363612920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:13,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:13,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363613119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:13,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363613129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:13,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T22:19:13,190 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:13,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:19:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-11T22:19:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T22:19:13,204 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:19:13,208 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:19:13,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:19:13,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/61289452fd154e7ead712eedf3fec970 2024-11-11T22:19:13,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/1dd857239cc6441fbeddf83658e7b94b is 50, key is test_row_0/C:col10/1731363551614/Put/seqid=0 2024-11-11T22:19:13,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742449_1625 (size=12301) 2024-11-11T22:19:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T22:19:13,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T22:19:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363613430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:13,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363613439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:13,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:13,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:13,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T22:19:13,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T22:19:13,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:13,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/1dd857239cc6441fbeddf83658e7b94b 2024-11-11T22:19:13,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/aac4d3222c0e47c8b7330b7d99518063 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/aac4d3222c0e47c8b7330b7d99518063 2024-11-11T22:19:13,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/aac4d3222c0e47c8b7330b7d99518063, entries=200, sequenceid=296, filesize=39.0 K 2024-11-11T22:19:13,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/61289452fd154e7ead712eedf3fec970 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/61289452fd154e7ead712eedf3fec970 2024-11-11T22:19:13,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T22:19:13,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/61289452fd154e7ead712eedf3fec970, entries=150, sequenceid=296, filesize=12.0 K 2024-11-11T22:19:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:13,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:13,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/1dd857239cc6441fbeddf83658e7b94b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/1dd857239cc6441fbeddf83658e7b94b 2024-11-11T22:19:13,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/1dd857239cc6441fbeddf83658e7b94b, entries=150, sequenceid=296, filesize=12.0 K 2024-11-11T22:19:13,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e0c62c1438a42621889aef2f44898dc9 in 913ms, sequenceid=296, compaction requested=true 2024-11-11T22:19:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:19:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:13,684 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:13,684 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:13,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:19:13,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:13,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:19:13,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:13,685 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:13,685 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:13,685 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/A is initiating minor compaction (all files) 2024-11-11T22:19:13,685 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/B is initiating minor compaction (all files) 2024-11-11T22:19:13,685 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/A in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,685 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/B in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,686 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/2a563a4dac7244bca1353efc1c8f2e09, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fcdc4e1d90eb4701bac24ee2cf5c4138, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/aac4d3222c0e47c8b7330b7d99518063] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=100.6 K 2024-11-11T22:19:13,686 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/ccf0cf34a4004a719e729f43d40db882, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/babbd5c20f5d40a0bdcaac83e8e8bca9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/61289452fd154e7ead712eedf3fec970] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=36.6 K 2024-11-11T22:19:13,686 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,686 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/2a563a4dac7244bca1353efc1c8f2e09, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fcdc4e1d90eb4701bac24ee2cf5c4138, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/aac4d3222c0e47c8b7330b7d99518063] 2024-11-11T22:19:13,686 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting ccf0cf34a4004a719e729f43d40db882, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363549693 2024-11-11T22:19:13,686 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a563a4dac7244bca1353efc1c8f2e09, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363549693 2024-11-11T22:19:13,686 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting babbd5c20f5d40a0bdcaac83e8e8bca9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731363550901 2024-11-11T22:19:13,686 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcdc4e1d90eb4701bac24ee2cf5c4138, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731363550901 2024-11-11T22:19:13,686 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 61289452fd154e7ead712eedf3fec970, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731363551614 2024-11-11T22:19:13,687 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac4d3222c0e47c8b7330b7d99518063, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731363551614 2024-11-11T22:19:13,694 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:13,695 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118060372d2f1e425d9a83650e0d52f5f4_e0c62c1438a42621889aef2f44898dc9 store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:13,697 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#B#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:13,697 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1f16b48c84f54fdea90d6b7fad8f6b2f is 50, key is test_row_0/B:col10/1731363551614/Put/seqid=0 2024-11-11T22:19:13,698 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118060372d2f1e425d9a83650e0d52f5f4_e0c62c1438a42621889aef2f44898dc9, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:13,698 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118060372d2f1e425d9a83650e0d52f5f4_e0c62c1438a42621889aef2f44898dc9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:13,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742451_1627 (size=4469) 2024-11-11T22:19:13,719 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#A#compaction#528 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:13,720 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/8f472e7bcd76459991fb105ae2986886 is 175, key is test_row_0/A:col10/1731363551614/Put/seqid=0 2024-11-11T22:19:13,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742450_1626 (size=13051) 2024-11-11T22:19:13,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742452_1628 (size=32005) 2024-11-11T22:19:13,748 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/1f16b48c84f54fdea90d6b7fad8f6b2f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1f16b48c84f54fdea90d6b7fad8f6b2f 2024-11-11T22:19:13,777 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/8f472e7bcd76459991fb105ae2986886 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/8f472e7bcd76459991fb105ae2986886 2024-11-11T22:19:13,781 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/B of e0c62c1438a42621889aef2f44898dc9 into 1f16b48c84f54fdea90d6b7fad8f6b2f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:13,781 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:13,781 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/B, priority=13, startTime=1731363553684; duration=0sec 2024-11-11T22:19:13,782 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:13,782 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:B 2024-11-11T22:19:13,782 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:13,783 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:13,783 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/C is initiating minor compaction (all files) 2024-11-11T22:19:13,784 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/C in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,784 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/58472634186f4f8b8a6cba1123550ec8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/22af7cab1cfc4e8daee27793d7edb232, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/1dd857239cc6441fbeddf83658e7b94b] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=36.6 K 2024-11-11T22:19:13,786 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 58472634186f4f8b8a6cba1123550ec8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731363549693 2024-11-11T22:19:13,787 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 22af7cab1cfc4e8daee27793d7edb232, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731363550901 2024-11-11T22:19:13,787 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dd857239cc6441fbeddf83658e7b94b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731363551614 2024-11-11T22:19:13,790 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/A of e0c62c1438a42621889aef2f44898dc9 into 8f472e7bcd76459991fb105ae2986886(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:13,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:13,790 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/A, priority=13, startTime=1731363553684; duration=0sec 2024-11-11T22:19:13,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:13,790 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:A 2024-11-11T22:19:13,806 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#C#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:13,807 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/8b8dae5d06764bf181522cc23b0e41d0 is 50, key is test_row_0/C:col10/1731363551614/Put/seqid=0 2024-11-11T22:19:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T22:19:13,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742453_1629 (size=13051) 2024-11-11T22:19:13,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T22:19:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:13,833 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T22:19:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:13,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118e9618c3565f49539f7944e4e7312826_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363552800/Put/seqid=0 2024-11-11T22:19:13,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742454_1630 (size=12454) 2024-11-11T22:19:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:14,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363614006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363614011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363614125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363614125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,238 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/8b8dae5d06764bf181522cc23b0e41d0 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/8b8dae5d06764bf181522cc23b0e41d0 2024-11-11T22:19:14,251 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/C of e0c62c1438a42621889aef2f44898dc9 into 8b8dae5d06764bf181522cc23b0e41d0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:14,251 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:14,251 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/C, priority=13, startTime=1731363553685; duration=0sec 2024-11-11T22:19:14,251 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:14,251 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:C 2024-11-11T22:19:14,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:14,318 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118e9618c3565f49539f7944e4e7312826_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118e9618c3565f49539f7944e4e7312826_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:14,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/504c3a08e0e249098670c975137bc723, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/504c3a08e0e249098670c975137bc723 is 175, key is test_row_0/A:col10/1731363552800/Put/seqid=0 2024-11-11T22:19:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T22:19:14,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363614329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742455_1631 (size=31255) 2024-11-11T22:19:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,334 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/504c3a08e0e249098670c975137bc723 2024-11-11T22:19:14,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363614332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/f37590f3a99042eaa8df8aaf416aad56 is 50, key is test_row_0/B:col10/1731363552800/Put/seqid=0 2024-11-11T22:19:14,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742456_1632 (size=12301) 2024-11-11T22:19:14,377 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/f37590f3a99042eaa8df8aaf416aad56 2024-11-11T22:19:14,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/97dce53af20b417eaa94a858538fb0cf is 50, key is test_row_0/C:col10/1731363552800/Put/seqid=0 2024-11-11T22:19:14,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742457_1633 (size=12301) 2024-11-11T22:19:14,418 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/97dce53af20b417eaa94a858538fb0cf 2024-11-11T22:19:14,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/504c3a08e0e249098670c975137bc723 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/504c3a08e0e249098670c975137bc723 2024-11-11T22:19:14,496 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/504c3a08e0e249098670c975137bc723, entries=150, sequenceid=310, filesize=30.5 K 2024-11-11T22:19:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/f37590f3a99042eaa8df8aaf416aad56 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/f37590f3a99042eaa8df8aaf416aad56 2024-11-11T22:19:14,534 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/f37590f3a99042eaa8df8aaf416aad56, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T22:19:14,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/97dce53af20b417eaa94a858538fb0cf as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/97dce53af20b417eaa94a858538fb0cf 2024-11-11T22:19:14,557 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/97dce53af20b417eaa94a858538fb0cf, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T22:19:14,557 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e0c62c1438a42621889aef2f44898dc9 in 724ms, sequenceid=310, compaction requested=false 2024-11-11T22:19:14,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:14,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:14,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-11T22:19:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-11T22:19:14,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-11T22:19:14,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3510 sec 2024-11-11T22:19:14,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.3710 sec 2024-11-11T22:19:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:14,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T22:19:14,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:14,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:19:14,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:14,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:19:14,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:14,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T22:19:14,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111102adb06c3adb49ad8da08e5a1ab3446f_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363554005/Put/seqid=0 2024-11-11T22:19:14,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742458_1634 (size=14994) 2024-11-11T22:19:14,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:14,663 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111102adb06c3adb49ad8da08e5a1ab3446f_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102adb06c3adb49ad8da08e5a1ab3446f_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:14,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/a20a42e4727d409f87c3c4820d0a9d58, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:14,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/a20a42e4727d409f87c3c4820d0a9d58 is 175, key is test_row_0/A:col10/1731363554005/Put/seqid=0 2024-11-11T22:19:14,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742459_1635 (size=39949) 2024-11-11T22:19:14,668 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/a20a42e4727d409f87c3c4820d0a9d58 2024-11-11T22:19:14,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/a2540a5821434c18b28f07dda9927040 is 50, key is test_row_0/B:col10/1731363554005/Put/seqid=0 2024-11-11T22:19:14,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363614679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363614680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742460_1636 (size=12301) 2024-11-11T22:19:14,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/a2540a5821434c18b28f07dda9927040 2024-11-11T22:19:14,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/d860a765f48e474f893a6dfaa14c3569 is 50, key is test_row_0/C:col10/1731363554005/Put/seqid=0 2024-11-11T22:19:14,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742461_1637 (size=12301) 2024-11-11T22:19:14,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363614789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363614789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363614999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363615000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/d860a765f48e474f893a6dfaa14c3569 2024-11-11T22:19:15,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/a20a42e4727d409f87c3c4820d0a9d58 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/a20a42e4727d409f87c3c4820d0a9d58 2024-11-11T22:19:15,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/a20a42e4727d409f87c3c4820d0a9d58, entries=200, sequenceid=332, filesize=39.0 K 2024-11-11T22:19:15,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/a2540a5821434c18b28f07dda9927040 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/a2540a5821434c18b28f07dda9927040 2024-11-11T22:19:15,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/a2540a5821434c18b28f07dda9927040, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T22:19:15,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/d860a765f48e474f893a6dfaa14c3569 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d860a765f48e474f893a6dfaa14c3569 2024-11-11T22:19:15,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d860a765f48e474f893a6dfaa14c3569, entries=150, sequenceid=332, filesize=12.0 K 2024-11-11T22:19:15,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for e0c62c1438a42621889aef2f44898dc9 in 506ms, sequenceid=332, compaction requested=true 2024-11-11T22:19:15,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:15,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T22:19:15,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:15,150 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:15,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T22:19:15,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:15,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e0c62c1438a42621889aef2f44898dc9:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T22:19:15,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T22:19:15,150 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:15,151 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:15,151 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/B is initiating minor compaction (all files) 2024-11-11T22:19:15,151 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/B in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,151 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1f16b48c84f54fdea90d6b7fad8f6b2f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/f37590f3a99042eaa8df8aaf416aad56, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/a2540a5821434c18b28f07dda9927040] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=36.8 K 2024-11-11T22:19:15,152 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f16b48c84f54fdea90d6b7fad8f6b2f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731363551614 2024-11-11T22:19:15,152 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting f37590f3a99042eaa8df8aaf416aad56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731363552787 2024-11-11T22:19:15,153 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting a2540a5821434c18b28f07dda9927040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731363553977 2024-11-11T22:19:15,161 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:15,161 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/A is initiating minor compaction (all files) 2024-11-11T22:19:15,161 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/A in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,161 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#B#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:15,161 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/8f472e7bcd76459991fb105ae2986886, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/504c3a08e0e249098670c975137bc723, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/a20a42e4727d409f87c3c4820d0a9d58] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=100.8 K 2024-11-11T22:19:15,161 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,161 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. files: [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/8f472e7bcd76459991fb105ae2986886, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/504c3a08e0e249098670c975137bc723, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/a20a42e4727d409f87c3c4820d0a9d58] 2024-11-11T22:19:15,162 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/34e64cc6957c476d8079bea22fb26f7d is 50, key is test_row_0/B:col10/1731363554005/Put/seqid=0 2024-11-11T22:19:15,162 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f472e7bcd76459991fb105ae2986886, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731363551614 2024-11-11T22:19:15,163 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting 504c3a08e0e249098670c975137bc723, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731363552787 2024-11-11T22:19:15,163 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] compactions.Compactor(225): Compacting a20a42e4727d409f87c3c4820d0a9d58, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731363553977 2024-11-11T22:19:15,182 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:15,191 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411119bd9fdd59dc34c2496f8d9df4c55feb8_e0c62c1438a42621889aef2f44898dc9 store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:15,193 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411119bd9fdd59dc34c2496f8d9df4c55feb8_e0c62c1438a42621889aef2f44898dc9, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:15,194 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119bd9fdd59dc34c2496f8d9df4c55feb8_e0c62c1438a42621889aef2f44898dc9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:15,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742462_1638 (size=13153) 2024-11-11T22:19:15,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742463_1639 (size=4469) 2024-11-11T22:19:15,231 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#A#compaction#538 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:15,231 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/64a56505f92843d2b2764034117c2450 is 175, key is test_row_0/A:col10/1731363554005/Put/seqid=0 2024-11-11T22:19:15,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742464_1640 (size=32107) 2024-11-11T22:19:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:15,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T22:19:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:15,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:15,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111522c1c9dca644aa5a06efc883fabad23_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363554676/Put/seqid=0 2024-11-11T22:19:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T22:19:15,340 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:15,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-11T22:19:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-11T22:19:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T22:19:15,342 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T22:19:15,342 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T22:19:15,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T22:19:15,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742465_1641 (size=14994) 2024-11-11T22:19:15,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363615359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363615362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T22:19:15,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363615470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363615470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,595 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/34e64cc6957c476d8079bea22fb26f7d as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/34e64cc6957c476d8079bea22fb26f7d 2024-11-11T22:19:15,599 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/B of e0c62c1438a42621889aef2f44898dc9 into 34e64cc6957c476d8079bea22fb26f7d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:15,599 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:15,599 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/B, priority=13, startTime=1731363555150; duration=0sec 2024-11-11T22:19:15,599 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T22:19:15,599 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:B 2024-11-11T22:19:15,599 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T22:19:15,600 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T22:19:15,600 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1541): e0c62c1438a42621889aef2f44898dc9/C is initiating minor compaction (all files) 2024-11-11T22:19:15,600 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e0c62c1438a42621889aef2f44898dc9/C in TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,600 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/8b8dae5d06764bf181522cc23b0e41d0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/97dce53af20b417eaa94a858538fb0cf, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d860a765f48e474f893a6dfaa14c3569] into tmpdir=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp, totalSize=36.8 K 2024-11-11T22:19:15,600 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b8dae5d06764bf181522cc23b0e41d0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731363551614 2024-11-11T22:19:15,600 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting 97dce53af20b417eaa94a858538fb0cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731363552787 2024-11-11T22:19:15,600 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] compactions.Compactor(225): Compacting d860a765f48e474f893a6dfaa14c3569, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731363553977 2024-11-11T22:19:15,605 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e0c62c1438a42621889aef2f44898dc9#C#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T22:19:15,606 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/d628ad819d494595be677d6caf1cb735 is 50, key is test_row_0/C:col10/1731363554005/Put/seqid=0 2024-11-11T22:19:15,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742466_1642 (size=13153) 2024-11-11T22:19:15,612 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/d628ad819d494595be677d6caf1cb735 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d628ad819d494595be677d6caf1cb735 2024-11-11T22:19:15,621 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/C of e0c62c1438a42621889aef2f44898dc9 into d628ad819d494595be677d6caf1cb735(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:15,622 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:15,622 INFO [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/C, priority=13, startTime=1731363555150; duration=0sec 2024-11-11T22:19:15,622 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:15,622 DEBUG [RS:0;8d82b89dd27d:36911-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:C 2024-11-11T22:19:15,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:15,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:15,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,650 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/64a56505f92843d2b2764034117c2450 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/64a56505f92843d2b2764034117c2450 2024-11-11T22:19:15,654 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e0c62c1438a42621889aef2f44898dc9/A of e0c62c1438a42621889aef2f44898dc9 into 64a56505f92843d2b2764034117c2450(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T22:19:15,654 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:15,654 INFO [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., storeName=e0c62c1438a42621889aef2f44898dc9/A, priority=13, startTime=1731363555149; duration=0sec 2024-11-11T22:19:15,654 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T22:19:15,654 DEBUG [RS:0;8d82b89dd27d:36911-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e0c62c1438a42621889aef2f44898dc9:A 2024-11-11T22:19:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T22:19:15,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363615679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363615679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:15,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111522c1c9dca644aa5a06efc883fabad23_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111522c1c9dca644aa5a06efc883fabad23_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:15,752 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/48f4bb28e12a4ab0b5fcc207171ccccb, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:15,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/48f4bb28e12a4ab0b5fcc207171ccccb is 175, key is test_row_0/A:col10/1731363554676/Put/seqid=0 2024-11-11T22:19:15,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742467_1643 (size=39949) 2024-11-11T22:19:15,763 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/48f4bb28e12a4ab0b5fcc207171ccccb 2024-11-11T22:19:15,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/5db4097f5fd24666992d711be12f67e3 is 50, key is test_row_0/B:col10/1731363554676/Put/seqid=0 2024-11-11T22:19:15,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742468_1644 (size=12301) 2024-11-11T22:19:15,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:15,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:15,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T22:19:15,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363615989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363615989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:16,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:16,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:16,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,161 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-11T22:19:16,161 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:16,161 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,161 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:16,163 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-11T22:19:16,163 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:16,163 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,163 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:16,164 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-11-11T22:19:16,164 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:16,164 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,164 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:16,166 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-11T22:19:16,166 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:16,166 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,166 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:16,167 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-11T22:19:16,167 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:16,167 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,167 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:16,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:16,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/5db4097f5fd24666992d711be12f67e3 2024-11-11T22:19:16,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/21da01d4a9564391a6a746d0335a0619 is 50, key is test_row_0/C:col10/1731363554676/Put/seqid=0 2024-11-11T22:19:16,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742469_1645 (size=12301) 2024-11-11T22:19:16,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T22:19:16,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:16,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-11T22:19:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50840 deadline: 1731363616499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50806 deadline: 1731363616499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 , the old value is region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T22:19:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., hostname=8d82b89dd27d,36911,1731363387997, seqNum=5 because the exception is null or not the one we care about 2024-11-11T22:19:16,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T22:19:16,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/21da01d4a9564391a6a746d0335a0619 2024-11-11T22:19:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/48f4bb28e12a4ab0b5fcc207171ccccb as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/48f4bb28e12a4ab0b5fcc207171ccccb 2024-11-11T22:19:16,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/48f4bb28e12a4ab0b5fcc207171ccccb, entries=200, sequenceid=348, filesize=39.0 K 2024-11-11T22:19:16,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/5db4097f5fd24666992d711be12f67e3 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/5db4097f5fd24666992d711be12f67e3 2024-11-11T22:19:16,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/5db4097f5fd24666992d711be12f67e3, entries=150, sequenceid=348, filesize=12.0 K 2024-11-11T22:19:16,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/21da01d4a9564391a6a746d0335a0619 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/21da01d4a9564391a6a746d0335a0619 2024-11-11T22:19:16,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/21da01d4a9564391a6a746d0335a0619, entries=150, sequenceid=348, filesize=12.0 K 2024-11-11T22:19:16,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e0c62c1438a42621889aef2f44898dc9 in 1278ms, sequenceid=348, compaction requested=false 2024-11-11T22:19:16,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:16,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36911 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T22:19:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:16,719 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T22:19:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:16,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:16,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a775ee1eea874f4486d08fc774c4f40c_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363555361/Put/seqid=0 2024-11-11T22:19:16,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742470_1646 (size=12454) 2024-11-11T22:19:17,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:17,129 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a775ee1eea874f4486d08fc774c4f40c_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a775ee1eea874f4486d08fc774c4f40c_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:17,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1e14fd69422240d6bc0009edb85fdd54, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:17,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1e14fd69422240d6bc0009edb85fdd54 is 175, key is test_row_0/A:col10/1731363555361/Put/seqid=0 2024-11-11T22:19:17,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742471_1647 (size=31255) 2024-11-11T22:19:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36911 {}] regionserver.HRegion(8855): Flush requested on e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:17,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. as already flushing 2024-11-11T22:19:17,240 INFO [Thread-2394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2394. 2024-11-11T22:19:17,240 DEBUG [Thread-2394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:17,240 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,241 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:17,310 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-11-11T22:19:17,310 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:17,310 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,310 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:17,321 INFO [Thread-2396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2396. 2024-11-11T22:19:17,321 DEBUG [Thread-2396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:17,321 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,321 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T22:19:17,510 INFO [Thread-2392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2392. 2024-11-11T22:19:17,510 DEBUG [Thread-2392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:17,510 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,510 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:17,510 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-11-11T22:19:17,511 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T22:19:17,511 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,511 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:17,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:17,534 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1e14fd69422240d6bc0009edb85fdd54 2024-11-11T22:19:17,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/15291ffe06204b3bb4a448d49b5d8e11 is 50, key is test_row_0/B:col10/1731363555361/Put/seqid=0 2024-11-11T22:19:17,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742472_1648 (size=12301) 2024-11-11T22:19:17,943 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/15291ffe06204b3bb4a448d49b5d8e11 2024-11-11T22:19:17,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/3c4ed2da281b48d9b6384ea0624f19e6 is 50, key is test_row_0/C:col10/1731363555361/Put/seqid=0 2024-11-11T22:19:17,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742473_1649 (size=12301) 2024-11-11T22:19:18,351 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/3c4ed2da281b48d9b6384ea0624f19e6 2024-11-11T22:19:18,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/1e14fd69422240d6bc0009edb85fdd54 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e14fd69422240d6bc0009edb85fdd54 2024-11-11T22:19:18,359 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e14fd69422240d6bc0009edb85fdd54, entries=150, sequenceid=367, filesize=30.5 K 2024-11-11T22:19:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/15291ffe06204b3bb4a448d49b5d8e11 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/15291ffe06204b3bb4a448d49b5d8e11 2024-11-11T22:19:18,362 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/15291ffe06204b3bb4a448d49b5d8e11, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T22:19:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/3c4ed2da281b48d9b6384ea0624f19e6 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/3c4ed2da281b48d9b6384ea0624f19e6 2024-11-11T22:19:18,365 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/3c4ed2da281b48d9b6384ea0624f19e6, entries=150, sequenceid=367, filesize=12.0 K 2024-11-11T22:19:18,366 INFO [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=33.54 KB/34350 for e0c62c1438a42621889aef2f44898dc9 in 1647ms, sequenceid=367, compaction requested=true 2024-11-11T22:19:18,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for e0c62c1438a42621889aef2f44898dc9: 2024-11-11T22:19:18,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:18,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8d82b89dd27d:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-11T22:19:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-11T22:19:18,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-11T22:19:18,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0250 sec 2024-11-11T22:19:18,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 3.0290 sec 2024-11-11T22:19:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T22:19:19,510 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2566 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2679 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2536 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2569 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2668 2024-11-11T22:19:19,510 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T22:19:19,510 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:19:19,510 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:19:19,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:19,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:19,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:19,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-11T22:19:19,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T22:19:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T22:19:19,514 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363559514"}]},"ts":"1731363559514"} 2024-11-11T22:19:19,516 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T22:19:19,516 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T22:19:19,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T22:19:19,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, UNASSIGN}] 2024-11-11T22:19:19,519 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, UNASSIGN 2024-11-11T22:19:19,521 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=e0c62c1438a42621889aef2f44898dc9, regionState=CLOSING, regionLocation=8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:19,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, UNASSIGN because future has completed 2024-11-11T22:19:19,523 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T22:19:19,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997}] 2024-11-11T22:19:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T22:19:19,675 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:19,675 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T22:19:19,675 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing e0c62c1438a42621889aef2f44898dc9, disabling compactions & flushes 2024-11-11T22:19:19,675 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:19,675 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:19,675 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. after waiting 0 ms 2024-11-11T22:19:19,675 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:19,675 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing e0c62c1438a42621889aef2f44898dc9 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T22:19:19,675 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=A 2024-11-11T22:19:19,675 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:19,676 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=B 2024-11-11T22:19:19,676 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:19,676 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e0c62c1438a42621889aef2f44898dc9, store=C 2024-11-11T22:19:19,676 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T22:19:19,680 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114c4193024d5445be918c971ed0860d56_e0c62c1438a42621889aef2f44898dc9 is 50, key is test_row_0/A:col10/1731363557239/Put/seqid=0 2024-11-11T22:19:19,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742474_1650 (size=12454) 2024-11-11T22:19:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T22:19:20,083 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T22:19:20,086 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114c4193024d5445be918c971ed0860d56_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c4193024d5445be918c971ed0860d56_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:20,087 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/dbe534cd7c76468b95f1906925299549, store: [table=TestAcidGuarantees family=A region=e0c62c1438a42621889aef2f44898dc9] 2024-11-11T22:19:20,088 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/dbe534cd7c76468b95f1906925299549 is 175, key is test_row_0/A:col10/1731363557239/Put/seqid=0 2024-11-11T22:19:20,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742475_1651 (size=31255) 2024-11-11T22:19:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T22:19:20,491 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/dbe534cd7c76468b95f1906925299549 2024-11-11T22:19:20,497 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/fb6a255bfcf943e5a60a036f4ede7f74 is 50, key is test_row_0/B:col10/1731363557239/Put/seqid=0 2024-11-11T22:19:20,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742476_1652 (size=12301) 2024-11-11T22:19:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T22:19:20,900 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/fb6a255bfcf943e5a60a036f4ede7f74 2024-11-11T22:19:20,906 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/354ad723af444804bf7f4450f7895e2e is 50, key is test_row_0/C:col10/1731363557239/Put/seqid=0 2024-11-11T22:19:20,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742477_1653 (size=12301) 2024-11-11T22:19:21,309 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/354ad723af444804bf7f4450f7895e2e 2024-11-11T22:19:21,312 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/A/dbe534cd7c76468b95f1906925299549 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dbe534cd7c76468b95f1906925299549 2024-11-11T22:19:21,315 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dbe534cd7c76468b95f1906925299549, entries=150, sequenceid=375, filesize=30.5 K 2024-11-11T22:19:21,315 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/B/fb6a255bfcf943e5a60a036f4ede7f74 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fb6a255bfcf943e5a60a036f4ede7f74 2024-11-11T22:19:21,318 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fb6a255bfcf943e5a60a036f4ede7f74, entries=150, sequenceid=375, filesize=12.0 K 2024-11-11T22:19:21,318 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/.tmp/C/354ad723af444804bf7f4450f7895e2e as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/354ad723af444804bf7f4450f7895e2e 2024-11-11T22:19:21,321 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/354ad723af444804bf7f4450f7895e2e, entries=150, sequenceid=375, filesize=12.0 K 2024-11-11T22:19:21,321 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e0c62c1438a42621889aef2f44898dc9 in 1646ms, sequenceid=375, compaction requested=true 2024-11-11T22:19:21,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e8c97566131481e8860c46b65b8dd77, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/272425726c454dfb82a760557de16dd8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/663a7343cf1c4ea6838bbdbd377c9084, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fb973258006044a8b2997f0477b819f2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/5440611d10a94051a2f9e1d70c39a342, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3c0b7adea1f941e68f53b32d2605b14c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/0ec18b67c91945ea8a9ab3ab7363fe40, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/909b7ba156ad4b9fb8781c58eee40e67, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1ec291d46cf845c580b0ead58ac3cdd2, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/828ff253807442b49c5251a1d8618220, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/cffc9a2c56b041949f85a93fb1d174fc, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/ff47d5407ec44fa8a2b703007c5bb0a5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3f04ac3bdf8c4d7c94dcb0c452694d8d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/61fa061dcfbf47a793142cb82f28c327, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dadd7a862d1f4a5784f5bd664e302497, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/e6909ae59fa340028054044d329932e7, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/b67dd23c2c7d452aad77c4185a93670b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/9023b4b6e7614bb3a56779dea7df2b74, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/21f68c9999424361a53a43b7699fc614, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/2a563a4dac7244bca1353efc1c8f2e09, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fcdc4e1d90eb4701bac24ee2cf5c4138, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/aac4d3222c0e47c8b7330b7d99518063, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/8f472e7bcd76459991fb105ae2986886, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/504c3a08e0e249098670c975137bc723, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/a20a42e4727d409f87c3c4820d0a9d58] to archive 2024-11-11T22:19:21,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:19:21,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e8c97566131481e8860c46b65b8dd77 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e8c97566131481e8860c46b65b8dd77 2024-11-11T22:19:21,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/272425726c454dfb82a760557de16dd8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/272425726c454dfb82a760557de16dd8 2024-11-11T22:19:21,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/663a7343cf1c4ea6838bbdbd377c9084 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/663a7343cf1c4ea6838bbdbd377c9084 2024-11-11T22:19:21,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fb973258006044a8b2997f0477b819f2 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fb973258006044a8b2997f0477b819f2 2024-11-11T22:19:21,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/5440611d10a94051a2f9e1d70c39a342 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/5440611d10a94051a2f9e1d70c39a342 2024-11-11T22:19:21,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3c0b7adea1f941e68f53b32d2605b14c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3c0b7adea1f941e68f53b32d2605b14c 2024-11-11T22:19:21,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/0ec18b67c91945ea8a9ab3ab7363fe40 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/0ec18b67c91945ea8a9ab3ab7363fe40 2024-11-11T22:19:21,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/909b7ba156ad4b9fb8781c58eee40e67 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/909b7ba156ad4b9fb8781c58eee40e67 2024-11-11T22:19:21,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1ec291d46cf845c580b0ead58ac3cdd2 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1ec291d46cf845c580b0ead58ac3cdd2 2024-11-11T22:19:21,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/828ff253807442b49c5251a1d8618220 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/828ff253807442b49c5251a1d8618220 2024-11-11T22:19:21,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/cffc9a2c56b041949f85a93fb1d174fc to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/cffc9a2c56b041949f85a93fb1d174fc 2024-11-11T22:19:21,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/ff47d5407ec44fa8a2b703007c5bb0a5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/ff47d5407ec44fa8a2b703007c5bb0a5 2024-11-11T22:19:21,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3f04ac3bdf8c4d7c94dcb0c452694d8d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/3f04ac3bdf8c4d7c94dcb0c452694d8d 2024-11-11T22:19:21,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/61fa061dcfbf47a793142cb82f28c327 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/61fa061dcfbf47a793142cb82f28c327 2024-11-11T22:19:21,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dadd7a862d1f4a5784f5bd664e302497 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dadd7a862d1f4a5784f5bd664e302497 2024-11-11T22:19:21,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/e6909ae59fa340028054044d329932e7 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/e6909ae59fa340028054044d329932e7 2024-11-11T22:19:21,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/b67dd23c2c7d452aad77c4185a93670b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/b67dd23c2c7d452aad77c4185a93670b 2024-11-11T22:19:21,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/9023b4b6e7614bb3a56779dea7df2b74 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/9023b4b6e7614bb3a56779dea7df2b74 2024-11-11T22:19:21,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/21f68c9999424361a53a43b7699fc614 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/21f68c9999424361a53a43b7699fc614 2024-11-11T22:19:21,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/2a563a4dac7244bca1353efc1c8f2e09 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/2a563a4dac7244bca1353efc1c8f2e09 2024-11-11T22:19:21,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fcdc4e1d90eb4701bac24ee2cf5c4138 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/fcdc4e1d90eb4701bac24ee2cf5c4138 2024-11-11T22:19:21,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/aac4d3222c0e47c8b7330b7d99518063 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/aac4d3222c0e47c8b7330b7d99518063 2024-11-11T22:19:21,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/8f472e7bcd76459991fb105ae2986886 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/8f472e7bcd76459991fb105ae2986886 2024-11-11T22:19:21,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/504c3a08e0e249098670c975137bc723 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/504c3a08e0e249098670c975137bc723 2024-11-11T22:19:21,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/a20a42e4727d409f87c3c4820d0a9d58 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/a20a42e4727d409f87c3c4820d0a9d58 2024-11-11T22:19:21,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1615484a4df44134b5fe908061cdd34a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1e9a3297e14744749000f2811ac90007, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/7a389d0c761b4ce3a21ac34710601714, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/aca9d746ab8b42c189180f71951b66a0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/af19c9ea95614212b51ffe6c53dcb759, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/c4ac9185c5794551b0248e8d2491b502, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d15058e3f6934b4581797007b44c429a, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fdf80effb804419e8f4bde3d4c8d829c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/4bf5c1b4b70d4bd4ad742ec650be89ee, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1c7a9751f0cb40479321d4d8f008ba2b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/0392852d336a4d91b9b98ebd04ee0db5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/91523b0b290b460ba3fbd5c7d750a503, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/b3c13e3d90ee4493b8bd70ec70843117, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/72d8b13d7dd148cb943bb68c809c2a6f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/11c9a2cce1c044ce8588bc69a4d49dca, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/8c14f18db42c438db68c93ea6c8bede3, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d524ac656b7a47e59ec18bf1839ce0fe, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1d5339b1bd814bb593ac2960044bde8e, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/ccf0cf34a4004a719e729f43d40db882, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/82d0970df9b144c89455cc6f66833c48, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/babbd5c20f5d40a0bdcaac83e8e8bca9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1f16b48c84f54fdea90d6b7fad8f6b2f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/61289452fd154e7ead712eedf3fec970, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/f37590f3a99042eaa8df8aaf416aad56, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/a2540a5821434c18b28f07dda9927040] to archive 2024-11-11T22:19:21,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:19:21,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1615484a4df44134b5fe908061cdd34a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1615484a4df44134b5fe908061cdd34a 2024-11-11T22:19:21,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1e9a3297e14744749000f2811ac90007 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1e9a3297e14744749000f2811ac90007 2024-11-11T22:19:21,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/7a389d0c761b4ce3a21ac34710601714 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/7a389d0c761b4ce3a21ac34710601714 2024-11-11T22:19:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/aca9d746ab8b42c189180f71951b66a0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/aca9d746ab8b42c189180f71951b66a0 2024-11-11T22:19:21,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/af19c9ea95614212b51ffe6c53dcb759 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/af19c9ea95614212b51ffe6c53dcb759 2024-11-11T22:19:21,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/c4ac9185c5794551b0248e8d2491b502 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/c4ac9185c5794551b0248e8d2491b502 2024-11-11T22:19:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d15058e3f6934b4581797007b44c429a to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d15058e3f6934b4581797007b44c429a 2024-11-11T22:19:21,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fdf80effb804419e8f4bde3d4c8d829c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fdf80effb804419e8f4bde3d4c8d829c 2024-11-11T22:19:21,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/4bf5c1b4b70d4bd4ad742ec650be89ee to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/4bf5c1b4b70d4bd4ad742ec650be89ee 2024-11-11T22:19:21,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1c7a9751f0cb40479321d4d8f008ba2b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1c7a9751f0cb40479321d4d8f008ba2b 2024-11-11T22:19:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/0392852d336a4d91b9b98ebd04ee0db5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/0392852d336a4d91b9b98ebd04ee0db5 2024-11-11T22:19:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/91523b0b290b460ba3fbd5c7d750a503 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/91523b0b290b460ba3fbd5c7d750a503 2024-11-11T22:19:21,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/b3c13e3d90ee4493b8bd70ec70843117 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/b3c13e3d90ee4493b8bd70ec70843117 2024-11-11T22:19:21,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/72d8b13d7dd148cb943bb68c809c2a6f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/72d8b13d7dd148cb943bb68c809c2a6f 2024-11-11T22:19:21,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/11c9a2cce1c044ce8588bc69a4d49dca to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/11c9a2cce1c044ce8588bc69a4d49dca 2024-11-11T22:19:21,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/8c14f18db42c438db68c93ea6c8bede3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/8c14f18db42c438db68c93ea6c8bede3 2024-11-11T22:19:21,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d524ac656b7a47e59ec18bf1839ce0fe to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/d524ac656b7a47e59ec18bf1839ce0fe 2024-11-11T22:19:21,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1d5339b1bd814bb593ac2960044bde8e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1d5339b1bd814bb593ac2960044bde8e 2024-11-11T22:19:21,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/ccf0cf34a4004a719e729f43d40db882 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/ccf0cf34a4004a719e729f43d40db882 2024-11-11T22:19:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/82d0970df9b144c89455cc6f66833c48 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/82d0970df9b144c89455cc6f66833c48 2024-11-11T22:19:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/babbd5c20f5d40a0bdcaac83e8e8bca9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/babbd5c20f5d40a0bdcaac83e8e8bca9 2024-11-11T22:19:21,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1f16b48c84f54fdea90d6b7fad8f6b2f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/1f16b48c84f54fdea90d6b7fad8f6b2f 2024-11-11T22:19:21,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/61289452fd154e7ead712eedf3fec970 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/61289452fd154e7ead712eedf3fec970 2024-11-11T22:19:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/f37590f3a99042eaa8df8aaf416aad56 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/f37590f3a99042eaa8df8aaf416aad56 2024-11-11T22:19:21,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/a2540a5821434c18b28f07dda9927040 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/a2540a5821434c18b28f07dda9927040 2024-11-11T22:19:21,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/86801e96e01046029baff540acd601ec, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/69b98701e25b4685a063e874f1a3d9f1, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/e114e0f67f9b4412a9f738d93d15362d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a84fb3043ac4416785dfede8ccc8dd3d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f9827784797f4c7e873599b6ae3b41e9, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61ad330168344bc89e69462dc128c484, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/0459a4fe73c54070b1b140793c9c688f, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/06889a60c73a46598eb88c4f2d064a50, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a9db1f70c9fd4c3fbd05c0f08d4fd08d, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/2d81a3a5c57645e9814b007ea4a855e8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bcfef6d7f0ec458c8108f7e1dee0ccb8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/81dbf73b78224da4a111034429c31fe6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/6341b6f0da4b46a2adc37554982d4467, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/473da8663fd9450cabaed5232f704ce5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/37402f3d14d349b08cf2aa7099441ebd, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/ff8dcdc2191f4ce3876d6a5d31f1f253, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bf00120e010e47b7993f61ee34c54e0c, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f5bd0ed55623432086332cd28e012fa6, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/58472634186f4f8b8a6cba1123550ec8, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61458fdae04d46d5924ab47bec6973e5, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/22af7cab1cfc4e8daee27793d7edb232, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/8b8dae5d06764bf181522cc23b0e41d0, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/1dd857239cc6441fbeddf83658e7b94b, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/97dce53af20b417eaa94a858538fb0cf, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d860a765f48e474f893a6dfaa14c3569] to archive 2024-11-11T22:19:21,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T22:19:21,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/86801e96e01046029baff540acd601ec to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/86801e96e01046029baff540acd601ec 2024-11-11T22:19:21,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/69b98701e25b4685a063e874f1a3d9f1 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/69b98701e25b4685a063e874f1a3d9f1 2024-11-11T22:19:21,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/e114e0f67f9b4412a9f738d93d15362d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/e114e0f67f9b4412a9f738d93d15362d 2024-11-11T22:19:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a84fb3043ac4416785dfede8ccc8dd3d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a84fb3043ac4416785dfede8ccc8dd3d 2024-11-11T22:19:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f9827784797f4c7e873599b6ae3b41e9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f9827784797f4c7e873599b6ae3b41e9 2024-11-11T22:19:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61ad330168344bc89e69462dc128c484 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61ad330168344bc89e69462dc128c484 2024-11-11T22:19:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/0459a4fe73c54070b1b140793c9c688f to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/0459a4fe73c54070b1b140793c9c688f 2024-11-11T22:19:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/06889a60c73a46598eb88c4f2d064a50 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/06889a60c73a46598eb88c4f2d064a50 2024-11-11T22:19:21,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a9db1f70c9fd4c3fbd05c0f08d4fd08d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/a9db1f70c9fd4c3fbd05c0f08d4fd08d 2024-11-11T22:19:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/2d81a3a5c57645e9814b007ea4a855e8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/2d81a3a5c57645e9814b007ea4a855e8 2024-11-11T22:19:21,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bcfef6d7f0ec458c8108f7e1dee0ccb8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bcfef6d7f0ec458c8108f7e1dee0ccb8 2024-11-11T22:19:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/81dbf73b78224da4a111034429c31fe6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/81dbf73b78224da4a111034429c31fe6 2024-11-11T22:19:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/6341b6f0da4b46a2adc37554982d4467 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/6341b6f0da4b46a2adc37554982d4467 2024-11-11T22:19:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/473da8663fd9450cabaed5232f704ce5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/473da8663fd9450cabaed5232f704ce5 2024-11-11T22:19:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/37402f3d14d349b08cf2aa7099441ebd to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/37402f3d14d349b08cf2aa7099441ebd 2024-11-11T22:19:21,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/ff8dcdc2191f4ce3876d6a5d31f1f253 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/ff8dcdc2191f4ce3876d6a5d31f1f253 2024-11-11T22:19:21,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bf00120e010e47b7993f61ee34c54e0c to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/bf00120e010e47b7993f61ee34c54e0c 2024-11-11T22:19:21,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f5bd0ed55623432086332cd28e012fa6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/f5bd0ed55623432086332cd28e012fa6 2024-11-11T22:19:21,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/58472634186f4f8b8a6cba1123550ec8 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/58472634186f4f8b8a6cba1123550ec8 2024-11-11T22:19:21,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61458fdae04d46d5924ab47bec6973e5 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/61458fdae04d46d5924ab47bec6973e5 2024-11-11T22:19:21,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/22af7cab1cfc4e8daee27793d7edb232 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/22af7cab1cfc4e8daee27793d7edb232 2024-11-11T22:19:21,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/8b8dae5d06764bf181522cc23b0e41d0 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/8b8dae5d06764bf181522cc23b0e41d0 2024-11-11T22:19:21,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/1dd857239cc6441fbeddf83658e7b94b to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/1dd857239cc6441fbeddf83658e7b94b 2024-11-11T22:19:21,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/97dce53af20b417eaa94a858538fb0cf to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/97dce53af20b417eaa94a858538fb0cf 2024-11-11T22:19:21,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d860a765f48e474f893a6dfaa14c3569 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d860a765f48e474f893a6dfaa14c3569 2024-11-11T22:19:21,385 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/recovered.edits/378.seqid, newMaxSeqId=378, maxSeqId=4 2024-11-11T22:19:21,385 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. 2024-11-11T22:19:21,385 DEBUG [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for e0c62c1438a42621889aef2f44898dc9: Waiting for close lock at 1731363559675Running coprocessor pre-close hooks at 1731363559675Disabling compacts and flushes for region at 1731363559675Disabling writes for close at 1731363559675Obtaining lock to block concurrent updates at 1731363559675Preparing flush snapshotting stores in e0c62c1438a42621889aef2f44898dc9 at 1731363559675Finished memstore snapshotting TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731363559676 (+1 ms)Flushing stores of TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9. at 1731363559676Flushing e0c62c1438a42621889aef2f44898dc9/A: creating writer at 1731363559676Flushing mob file e0c62c1438a42621889aef2f44898dc9/A: appending metadata at 1731363559680 (+4 ms)Flushing mob file e0c62c1438a42621889aef2f44898dc9/A: closing flushed file at 1731363559680Flushing e0c62c1438a42621889aef2f44898dc9/A: appending metadata at 1731363560087 (+407 ms)Flushing e0c62c1438a42621889aef2f44898dc9/A: closing flushed file at 1731363560087Flushing e0c62c1438a42621889aef2f44898dc9/B: creating writer at 1731363560494 (+407 ms)Flushing e0c62c1438a42621889aef2f44898dc9/B: appending metadata at 1731363560496 (+2 ms)Flushing e0c62c1438a42621889aef2f44898dc9/B: closing flushed file at 1731363560497 (+1 ms)Flushing e0c62c1438a42621889aef2f44898dc9/C: creating writer at 1731363560904 (+407 ms)Flushing e0c62c1438a42621889aef2f44898dc9/C: appending metadata at 1731363560905 (+1 ms)Flushing e0c62c1438a42621889aef2f44898dc9/C: closing flushed file at 1731363560906 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15a7fbc1: reopening flushed file at 1731363561312 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6699986a: reopening flushed file at 1731363561315 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48cc0746: reopening flushed file at 1731363561318 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e0c62c1438a42621889aef2f44898dc9 in 1646ms, sequenceid=375, compaction requested=true at 1731363561321 (+3 ms)Writing region close event to WAL at 1731363561382 (+61 ms)Running coprocessor post-close hooks at 1731363561385 (+3 ms)Closed at 1731363561385 2024-11-11T22:19:21,387 INFO [RS_CLOSE_REGION-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,387 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=e0c62c1438a42621889aef2f44898dc9, regionState=CLOSED 2024-11-11T22:19:21,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 because future has completed 2024-11-11T22:19:21,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-11T22:19:21,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure e0c62c1438a42621889aef2f44898dc9, server=8d82b89dd27d,36911,1731363387997 in 1.8660 sec 2024-11-11T22:19:21,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-11T22:19:21,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e0c62c1438a42621889aef2f44898dc9, UNASSIGN in 1.8720 sec 2024-11-11T22:19:21,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-11T22:19:21,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8760 sec 2024-11-11T22:19:21,394 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731363561394"}]},"ts":"1731363561394"} 2024-11-11T22:19:21,395 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T22:19:21,395 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T22:19:21,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8840 sec 2024-11-11T22:19:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T22:19:21,659 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:21,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-11T22:19:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:19:21,662 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:19:21,662 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:19:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T22:19:21,664 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,666 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C, FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/recovered.edits] 2024-11-11T22:19:21,668 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e14fd69422240d6bc0009edb85fdd54 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/1e14fd69422240d6bc0009edb85fdd54 2024-11-11T22:19:21,668 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/48f4bb28e12a4ab0b5fcc207171ccccb to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/48f4bb28e12a4ab0b5fcc207171ccccb 2024-11-11T22:19:21,669 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/64a56505f92843d2b2764034117c2450 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/64a56505f92843d2b2764034117c2450 2024-11-11T22:19:21,670 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dbe534cd7c76468b95f1906925299549 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/A/dbe534cd7c76468b95f1906925299549 2024-11-11T22:19:21,671 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/15291ffe06204b3bb4a448d49b5d8e11 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/15291ffe06204b3bb4a448d49b5d8e11 2024-11-11T22:19:21,672 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/34e64cc6957c476d8079bea22fb26f7d to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/34e64cc6957c476d8079bea22fb26f7d 2024-11-11T22:19:21,673 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/5db4097f5fd24666992d711be12f67e3 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/5db4097f5fd24666992d711be12f67e3 2024-11-11T22:19:21,674 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fb6a255bfcf943e5a60a036f4ede7f74 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/B/fb6a255bfcf943e5a60a036f4ede7f74 2024-11-11T22:19:21,675 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/21da01d4a9564391a6a746d0335a0619 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/21da01d4a9564391a6a746d0335a0619 2024-11-11T22:19:21,676 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/354ad723af444804bf7f4450f7895e2e to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/354ad723af444804bf7f4450f7895e2e 2024-11-11T22:19:21,676 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/3c4ed2da281b48d9b6384ea0624f19e6 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/3c4ed2da281b48d9b6384ea0624f19e6 2024-11-11T22:19:21,677 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d628ad819d494595be677d6caf1cb735 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/C/d628ad819d494595be677d6caf1cb735 2024-11-11T22:19:21,678 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/recovered.edits/378.seqid to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9/recovered.edits/378.seqid 2024-11-11T22:19:21,679 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/default/TestAcidGuarantees/e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,679 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T22:19:21,679 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:19:21,680 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T22:19:21,682 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102adb06c3adb49ad8da08e5a1ab3446f_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111102adb06c3adb49ad8da08e5a1ab3446f_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,682 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111112043c3c8cd3439f94156a36dd5279c0_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111112043c3c8cd3439f94156a36dd5279c0_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,683 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111124fc5ca344549deb495afb93b7f0c4a_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111124fc5ca344549deb495afb93b7f0c4a_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,684 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111129afd8c514542dc9aba19bc1e561187_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111129afd8c514542dc9aba19bc1e561187_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,685 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111ee83789a8074209a886ce1bf4b0c91d_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111ee83789a8074209a886ce1bf4b0c91d_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,685 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112dfc4b2b8c634248a8bc0e8e027b59d8_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112dfc4b2b8c634248a8bc0e8e027b59d8_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,686 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111332d1a7277b8458ebb2d2b8d1004eb3f_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111332d1a7277b8458ebb2d2b8d1004eb3f_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,687 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c4193024d5445be918c971ed0860d56_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c4193024d5445be918c971ed0860d56_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,688 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115135d078b1f941db8f7a9787069dc6e9_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115135d078b1f941db8f7a9787069dc6e9_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,689 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111522c1c9dca644aa5a06efc883fabad23_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111522c1c9dca644aa5a06efc883fabad23_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,689 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117e0385e0a20c4d64800aeee77fc497cf_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117e0385e0a20c4d64800aeee77fc497cf_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,690 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118e9618c3565f49539f7944e4e7312826_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118e9618c3565f49539f7944e4e7312826_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,692 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a75a8f94817e432cbfb16657f09e0ca5_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a75a8f94817e432cbfb16657f09e0ca5_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,693 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a775ee1eea874f4486d08fc774c4f40c_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a775ee1eea874f4486d08fc774c4f40c_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,694 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bcfd5d5074de4115b6ad5eff64e3b8e2_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bcfd5d5074de4115b6ad5eff64e3b8e2_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,695 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0ff349e8dee4953a56c52e5f917d385_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c0ff349e8dee4953a56c52e5f917d385_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,696 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc95f630d7e84dbbaffe1273093c8b0f_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111cc95f630d7e84dbbaffe1273093c8b0f_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,697 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5b22c3c2695429cbfdf4152ff29b233_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5b22c3c2695429cbfdf4152ff29b233_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,698 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5f8f18c38c74bf6a6b841530670d8ee_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5f8f18c38c74bf6a6b841530670d8ee_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,698 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e974a5da950347c1aa9b72ed49c03c31_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e974a5da950347c1aa9b72ed49c03c31_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,700 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ff3bb5e47b624fdaa07ec827b99074aa_e0c62c1438a42621889aef2f44898dc9 to hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ff3bb5e47b624fdaa07ec827b99074aa_e0c62c1438a42621889aef2f44898dc9 2024-11-11T22:19:21,701 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T22:19:21,703 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:19:21,705 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T22:19:21,707 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T22:19:21,708 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:19:21,709 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T22:19:21,709 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731363561709"}]},"ts":"9223372036854775807"} 2024-11-11T22:19:21,710 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T22:19:21,710 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e0c62c1438a42621889aef2f44898dc9, NAME => 'TestAcidGuarantees,,1731363533035.e0c62c1438a42621889aef2f44898dc9.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T22:19:21,710 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T22:19:21,711 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731363561711"}]},"ts":"9223372036854775807"} 2024-11-11T22:19:21,712 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T22:19:21,713 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T22:19:21,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-11T22:19:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41919 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T22:19:21,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T22:19:21,769 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T22:19:21,777 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=639 (was 578) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=778 (was 878) 2024-11-11T22:19:21,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T22:19:21,778 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:19:21,778 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:19:21,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:21,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:21,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T22:19:21,778 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T22:19:21,778 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-11T22:19:21,778 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8d82b89dd27d,41919,1731363387169 2024-11-11T22:19:21,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T22:19:21,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T22:19:21,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:19:21,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:19:21,825 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T22:19:21,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T22:19:21,826 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T22:19:21,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:19:21,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:21,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T22:19:21,827 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8d82b89dd27d,36911,1731363387997' ***** 2024-11-11T22:19:21,827 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T22:19:21,827 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T22:19:21,827 INFO [RS:0;8d82b89dd27d:36911 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T22:19:21,827 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T22:19:21,827 INFO [RS:0;8d82b89dd27d:36911 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T22:19:21,827 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(959): stopping server 8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:21,828 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T22:19:21,828 INFO [RS:0;8d82b89dd27d:36911 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8d82b89dd27d:36911. 2024-11-11T22:19:21,828 DEBUG [RS:0;8d82b89dd27d:36911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T22:19:21,828 DEBUG [RS:0;8d82b89dd27d:36911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:21,828 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T22:19:21,828 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T22:19:21,828 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T22:19:21,828 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T22:19:21,829 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T22:19:21,829 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T22:19:21,829 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:21,829 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T22:19:21,829 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T22:19:21,829 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T22:19:21,829 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T22:19:21,829 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T22:19:21,829 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T22:19:21,851 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/info/768c5d7345644715afb9e1c05f0fa523 is 95, key is TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391./info:/1731363421613/DeleteFamily/seqid=0 2024-11-11T22:19:21,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742478_1654 (size=6514) 2024-11-11T22:19:21,880 INFO [regionserver/8d82b89dd27d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T22:19:22,029 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:22,120 INFO [regionserver/8d82b89dd27d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T22:19:22,121 INFO [regionserver/8d82b89dd27d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T22:19:22,230 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:22,254 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/info/768c5d7345644715afb9e1c05f0fa523 2024-11-11T22:19:22,275 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/ns/b0fcf66c9a814883929debe99104785f is 93, key is TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391./ns:/1731363421519/DeleteFamily/seqid=0 2024-11-11T22:19:22,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742479_1655 (size=5906) 2024-11-11T22:19:22,430 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:22,630 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:22,679 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/ns/b0fcf66c9a814883929debe99104785f 2024-11-11T22:19:22,698 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/rep_barrier/aea25ff9d99d4abbb9134ce93dd7943b is 102, key is TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391./rep_barrier:/1731363421519/DeleteFamily/seqid=0 2024-11-11T22:19:22,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742480_1656 (size=6025) 2024-11-11T22:19:22,830 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:23,030 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:23,102 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/rep_barrier/aea25ff9d99d4abbb9134ce93dd7943b 2024-11-11T22:19:23,118 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/table/72802445e1cc48328c291f58c72872af is 96, key is TestAcidGuarantees,,1731363392652.3f60f99bf48ef670ca729411903c0391./table:/1731363421519/DeleteFamily/seqid=0 2024-11-11T22:19:23,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742481_1657 (size=5936) 2024-11-11T22:19:23,230 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:23,431 DEBUG [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T22:19:23,521 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/table/72802445e1cc48328c291f58c72872af 2024-11-11T22:19:23,525 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/info/768c5d7345644715afb9e1c05f0fa523 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/info/768c5d7345644715afb9e1c05f0fa523 2024-11-11T22:19:23,528 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/info/768c5d7345644715afb9e1c05f0fa523, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T22:19:23,528 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/ns/b0fcf66c9a814883929debe99104785f as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/ns/b0fcf66c9a814883929debe99104785f 2024-11-11T22:19:23,531 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/ns/b0fcf66c9a814883929debe99104785f, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T22:19:23,531 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/rep_barrier/aea25ff9d99d4abbb9134ce93dd7943b as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/rep_barrier/aea25ff9d99d4abbb9134ce93dd7943b 2024-11-11T22:19:23,533 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/rep_barrier/aea25ff9d99d4abbb9134ce93dd7943b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T22:19:23,534 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/.tmp/table/72802445e1cc48328c291f58c72872af as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/table/72802445e1cc48328c291f58c72872af 2024-11-11T22:19:23,536 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/table/72802445e1cc48328c291f58c72872af, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T22:19:23,537 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false 2024-11-11T22:19:23,542 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T22:19:23,543 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T22:19:23,543 INFO [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T22:19:23,543 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731363561829Running coprocessor pre-close hooks at 1731363561829Disabling compacts and flushes for region at 1731363561829Disabling writes for close at 1731363561829Obtaining lock to block concurrent updates at 1731363561829Preparing flush snapshotting stores in 1588230740 at 1731363561829Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731363561830 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731363561831 (+1 ms)Flushing 1588230740/info: creating writer at 1731363561831Flushing 1588230740/info: appending metadata at 1731363561851 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731363561851Flushing 1588230740/ns: creating writer at 1731363562258 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731363562275 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731363562275Flushing 1588230740/rep_barrier: creating writer at 1731363562682 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731363562698 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731363562698Flushing 1588230740/table: creating writer at 1731363563105 (+407 ms)Flushing 1588230740/table: appending metadata at 1731363563118 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731363563118Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13eeb42c: reopening flushed file at 1731363563524 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53057c3f: reopening flushed file at 1731363563528 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c070482: reopening flushed file at 1731363563531 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e924c51: reopening flushed file at 1731363563534 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false at 1731363563537 (+3 ms)Writing region close event to WAL at 1731363563540 (+3 ms)Running coprocessor post-close hooks at 1731363563543 (+3 ms)Closed at 1731363563543 2024-11-11T22:19:23,543 DEBUG [RS_CLOSE_META-regionserver/8d82b89dd27d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T22:19:23,631 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(976): stopping server 8d82b89dd27d,36911,1731363387997; all regions closed. 2024-11-11T22:19:23,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741834_1010 (size=25344) 2024-11-11T22:19:23,637 DEBUG [RS:0;8d82b89dd27d:36911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/oldWALs 2024-11-11T22:19:23,637 INFO [RS:0;8d82b89dd27d:36911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8d82b89dd27d%2C36911%2C1731363387997.meta:.meta(num 1731363392120) 2024-11-11T22:19:23,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741832_1008 (size=16678815) 2024-11-11T22:19:23,640 DEBUG [RS:0;8d82b89dd27d:36911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/oldWALs 2024-11-11T22:19:23,640 INFO [RS:0;8d82b89dd27d:36911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8d82b89dd27d%2C36911%2C1731363387997:(num 1731363391181) 2024-11-11T22:19:23,640 DEBUG [RS:0;8d82b89dd27d:36911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T22:19:23,640 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T22:19:23,640 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T22:19:23,641 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.ChoreService(370): Chore service for: regionserver/8d82b89dd27d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T22:19:23,641 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T22:19:23,641 INFO [regionserver/8d82b89dd27d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T22:19:23,642 INFO [RS:0;8d82b89dd27d:36911 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36911 2024-11-11T22:19:23,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T22:19:23,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8d82b89dd27d,36911,1731363387997 2024-11-11T22:19:23,699 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T22:19:23,709 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8d82b89dd27d,36911,1731363387997] 2024-11-11T22:19:23,720 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8d82b89dd27d,36911,1731363387997 already deleted, retry=false 2024-11-11T22:19:23,720 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8d82b89dd27d,36911,1731363387997 expired; onlineServers=0 2024-11-11T22:19:23,720 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8d82b89dd27d,41919,1731363387169' ***** 2024-11-11T22:19:23,720 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T22:19:23,720 INFO [M:0;8d82b89dd27d:41919 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T22:19:23,720 INFO [M:0;8d82b89dd27d:41919 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T22:19:23,720 DEBUG [M:0;8d82b89dd27d:41919 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T22:19:23,721 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T22:19:23,721 DEBUG [M:0;8d82b89dd27d:41919 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T22:19:23,721 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.small.0-1731363390820 {}] cleaner.HFileCleaner(306): Exit Thread[master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.small.0-1731363390820,5,FailOnTimeoutGroup] 2024-11-11T22:19:23,721 DEBUG [master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.large.0-1731363390818 {}] cleaner.HFileCleaner(306): Exit Thread[master/8d82b89dd27d:0:becomeActiveMaster-HFileCleaner.large.0-1731363390818,5,FailOnTimeoutGroup] 2024-11-11T22:19:23,721 INFO [M:0;8d82b89dd27d:41919 {}] hbase.ChoreService(370): Chore service for: master/8d82b89dd27d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T22:19:23,721 INFO [M:0;8d82b89dd27d:41919 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T22:19:23,721 DEBUG [M:0;8d82b89dd27d:41919 {}] master.HMaster(1795): Stopping service threads 2024-11-11T22:19:23,721 INFO [M:0;8d82b89dd27d:41919 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T22:19:23,721 INFO [M:0;8d82b89dd27d:41919 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T22:19:23,721 ERROR [M:0;8d82b89dd27d:41919 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1420519831) connection to localhost/127.0.0.1:37063 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37063,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T22:19:23,722 INFO [M:0;8d82b89dd27d:41919 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T22:19:23,722 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T22:19:23,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T22:19:23,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T22:19:23,731 DEBUG [M:0;8d82b89dd27d:41919 {}] zookeeper.ZKUtil(347): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T22:19:23,731 WARN [M:0;8d82b89dd27d:41919 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T22:19:23,731 INFO [M:0;8d82b89dd27d:41919 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/.lastflushedseqids 2024-11-11T22:19:23,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742482_1658 (size=72) 2024-11-11T22:19:23,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T22:19:23,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36911-0x1012bf8af6e0001, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T22:19:23,810 INFO [RS:0;8d82b89dd27d:36911 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T22:19:23,811 INFO [RS:0;8d82b89dd27d:36911 {}] regionserver.HRegionServer(1031): Exiting; stopping=8d82b89dd27d,36911,1731363387997; zookeeper connection closed. 2024-11-11T22:19:23,811 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@19de7127 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@19de7127 2024-11-11T22:19:23,811 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T22:19:24,139 INFO [M:0;8d82b89dd27d:41919 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T22:19:24,139 INFO [M:0;8d82b89dd27d:41919 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T22:19:24,139 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T22:19:24,139 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:19:24,139 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:19:24,139 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T22:19:24,139 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:19:24,139 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=777.23 KB heapSize=951.95 KB 2024-11-11T22:19:24,153 DEBUG [M:0;8d82b89dd27d:41919 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b94ef77d83c4408b31f7544d234c60c is 82, key is hbase:meta,,1/info:regioninfo/1731363392203/Put/seqid=0 2024-11-11T22:19:24,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742483_1659 (size=5672) 2024-11-11T22:19:24,556 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b94ef77d83c4408b31f7544d234c60c 2024-11-11T22:19:24,579 DEBUG [M:0;8d82b89dd27d:41919 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/656260c209714871bfb2f1690a911590 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731363536090/Put/seqid=0 2024-11-11T22:19:24,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742484_1660 (size=45774) 2024-11-11T22:19:24,986 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=776.67 KB at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/656260c209714871bfb2f1690a911590 2024-11-11T22:19:24,989 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 656260c209714871bfb2f1690a911590 2024-11-11T22:19:25,004 DEBUG [M:0;8d82b89dd27d:41919 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/312d2894d04e444e81841e21c65c72a5 is 69, key is 8d82b89dd27d,36911,1731363387997/rs:state/1731363390948/Put/seqid=0 2024-11-11T22:19:25,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742485_1661 (size=5156) 2024-11-11T22:19:25,407 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2155 (bloomFilter=true), to=hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/312d2894d04e444e81841e21c65c72a5 2024-11-11T22:19:25,411 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b94ef77d83c4408b31f7544d234c60c as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b94ef77d83c4408b31f7544d234c60c 2024-11-11T22:19:25,414 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b94ef77d83c4408b31f7544d234c60c, entries=8, sequenceid=2155, filesize=5.5 K 2024-11-11T22:19:25,415 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/656260c209714871bfb2f1690a911590 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/656260c209714871bfb2f1690a911590 2024-11-11T22:19:25,418 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 656260c209714871bfb2f1690a911590 2024-11-11T22:19:25,418 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/656260c209714871bfb2f1690a911590, entries=172, sequenceid=2155, filesize=44.7 K 2024-11-11T22:19:25,418 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/312d2894d04e444e81841e21c65c72a5 as hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/312d2894d04e444e81841e21c65c72a5 2024-11-11T22:19:25,420 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37063/user/jenkins/test-data/0c1b6ee2-e45b-7c83-fe5e-a357b22f9b9e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/312d2894d04e444e81841e21c65c72a5, entries=1, sequenceid=2155, filesize=5.0 K 2024-11-11T22:19:25,421 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(3140): Finished flush of dataSize ~777.23 KB/795879, heapSize ~951.65 KB/974488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2155, compaction requested=false 2024-11-11T22:19:25,422 INFO [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T22:19:25,422 DEBUG [M:0;8d82b89dd27d:41919 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731363564139Disabling compacts and flushes for region at 1731363564139Disabling writes for close at 1731363564139Obtaining lock to block concurrent updates at 1731363564140 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731363564140Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=795879, getHeapSize=974728, getOffHeapSize=0, getCellsCount=2352 at 1731363564140Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731363564140Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731363564140Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731363564153 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731363564153Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731363564560 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731363564578 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731363564578Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731363564989 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731363565003 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731363565003Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ffdb7ae: reopening flushed file at 1731363565411 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e185bc5: reopening flushed file at 1731363565414 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d4a057d: reopening flushed file at 1731363565418 (+4 ms)Finished flush of dataSize ~777.23 KB/795879, heapSize ~951.65 KB/974488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1282ms, sequenceid=2155, compaction requested=false at 1731363565421 (+3 ms)Writing region close event to WAL at 1731363565422 (+1 ms)Closed at 1731363565422 2024-11-11T22:19:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741830_1006 (size=937161) 2024-11-11T22:19:25,425 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T22:19:25,425 INFO [M:0;8d82b89dd27d:41919 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T22:19:25,425 INFO [M:0;8d82b89dd27d:41919 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41919 2024-11-11T22:19:25,425 INFO [M:0;8d82b89dd27d:41919 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T22:19:25,609 INFO [M:0;8d82b89dd27d:41919 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T22:19:25,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T22:19:25,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41919-0x1012bf8af6e0000, quorum=127.0.0.1:54105, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T22:19:25,614 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T22:19:25,616 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T22:19:25,616 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T22:19:25,616 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T22:19:25,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/hadoop.log.dir/,STOPPED} 2024-11-11T22:19:25,619 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T22:19:25,619 WARN [BP-2142985488-172.17.0.3-1731363382824 heartbeating to localhost/127.0.0.1:37063 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T22:19:25,619 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T22:19:25,619 WARN [BP-2142985488-172.17.0.3-1731363382824 heartbeating to localhost/127.0.0.1:37063 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2142985488-172.17.0.3-1731363382824 (Datanode Uuid 1380727f-fd36-4c3d-bafb-8685d0ba11cc) service to localhost/127.0.0.1:37063 2024-11-11T22:19:25,621 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/data/data1/current/BP-2142985488-172.17.0.3-1731363382824 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T22:19:25,622 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/cluster_d3af44ce-f1f7-3cdc-1123-d58935b065b8/data/data2/current/BP-2142985488-172.17.0.3-1731363382824 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T22:19:25,622 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T22:19:25,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T22:19:25,630 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T22:19:25,630 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T22:19:25,631 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T22:19:25,631 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/502bc689-75b4-524d-4383-84972c4f7949/hadoop.log.dir/,STOPPED} 2024-11-11T22:19:25,648 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T22:19:25,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down